2024-05-25T05:45:29.444 INFO:root:teuthology version: 0.0.1.dev291+ge7ae758 2024-05-25T05:45:29.445 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343 --owner scheduled_teuthology@teuthology --verbose --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} --name teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343/orig.config.yaml 2024-05-25T05:45:29.490 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-25T05:45:29.584 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} email: ceph-qa@ceph.com first_in_suite: false job_id: '7725343' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: ea0684c08cbcf83f4af9930de2267b3563ac5f78 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ea0684c08cbcf83f4af9930de2267b3563ac5f78 workunit: branch: squid sha1: 17c123e524b17eb0cf38bdb0a0ca89d80b36c131 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 4121 sha1: ea0684c08cbcf83f4af9930de2267b3563ac5f78 sleep_before_teardown: 0 subset: 40/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_17c123e524b17eb0cf38bdb0a0ca89d80b36c131/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 17c123e524b17eb0cf38bdb0a0ca89d80b36c131 targets: smithi175.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBN0/l8WXI4yiHJcAlkk41grHO7Suqu7TOQclfDtY5fdL5OxLdqa6cqqU2GpayhYFNRz6MUpVlz0B5Wh5iaqzB84= smithi179.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPn+EbiNJu847TU/w4/mBSLNfRuCFUOd/j2CRPLmDyzsM6DqkOTg0XSSVTPEyYD1SYO8i3OjF+nps6dGOHoQYTk= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e7ae7580c417d6ab27fda98fecadca5d9f85d63e timestamp: 2024-05-24_21:08:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 2024-05-25T05:45:29.585 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_17c123e524b17eb0cf38bdb0a0ca89d80b36c131/qa; will attempt to use it 2024-05-25T05:45:29.585 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_17c123e524b17eb0cf38bdb0a0ca89d80b36c131/qa/tasks 2024-05-25T05:45:29.586 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-25T05:45:29.587 INFO:teuthology.task.internal:Checking packages... 2024-05-25T05:45:29.608 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'ea0684c08cbcf83f4af9930de2267b3563ac5f78' 2024-05-25T05:45:29.608 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-25T05:45:29.609 INFO:teuthology.packaging:ref: None 2024-05-25T05:45:29.609 INFO:teuthology.packaging:tag: None 2024-05-25T05:45:29.609 INFO:teuthology.packaging:branch: squid 2024-05-25T05:45:29.609 INFO:teuthology.packaging:sha1: ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T05:45:29.609 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-05-25T05:45:29.865 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2847.g17c123e5 2024-05-25T05:45:29.867 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-25T05:45:29.873 INFO:teuthology.task.internal:no buildpackages task found 2024-05-25T05:45:29.873 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-25T05:45:29.892 INFO:teuthology.task.internal:Saving configuration 2024-05-25T05:45:29.909 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-25T05:45:29.916 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-25T05:45:29.941 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi175.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-25 05:40:34.061990', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBN0/l8WXI4yiHJcAlkk41grHO7Suqu7TOQclfDtY5fdL5OxLdqa6cqqU2GpayhYFNRz6MUpVlz0B5Wh5iaqzB84='} 2024-05-25T05:45:29.966 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi179.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-25 05:40:34.060396', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPn+EbiNJu847TU/w4/mBSLNfRuCFUOd/j2CRPLmDyzsM6DqkOTg0XSSVTPEyYD1SYO8i3OjF+nps6dGOHoQYTk='} 2024-05-25T05:45:29.966 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-25T05:45:29.972 INFO:teuthology.task.internal:roles: ubuntu@smithi175.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-05-25T05:45:29.972 INFO:teuthology.task.internal:roles: ubuntu@smithi179.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-05-25T05:45:29.972 INFO:teuthology.run_tasks:Running task console_log... 2024-05-25T05:45:30.035 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ffb5643fac0>, signals=[15]) 2024-05-25T05:45:30.036 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-25T05:45:30.042 INFO:teuthology.task.internal:Opening connections... 2024-05-25T05:45:30.042 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi175.front.sepia.ceph.com 2024-05-25T05:45:30.043 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:45:30.120 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi179.front.sepia.ceph.com 2024-05-25T05:45:30.121 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:45:30.193 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-25T05:45:30.202 DEBUG:teuthology.orchestra.run.smithi175:> uname -m 2024-05-25T05:45:30.220 INFO:teuthology.orchestra.run.smithi175.stdout:x86_64 2024-05-25T05:45:30.220 DEBUG:teuthology.orchestra.run.smithi175:> cat /etc/os-release 2024-05-25T05:45:30.276 INFO:teuthology.orchestra.run.smithi175.stdout:NAME="CentOS Stream" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:VERSION="9" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:ID="centos" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:ID_LIKE="rhel fedora" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:VERSION_ID="9" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:PLATFORM_ID="platform:el9" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:ANSI_COLOR="0;31" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:LOGO="fedora-logo-icon" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:HOME_URL="https://centos.org/" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-25T05:45:30.277 INFO:teuthology.orchestra.run.smithi175.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-25T05:45:30.278 INFO:teuthology.orchestra.run.smithi175.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-25T05:45:30.278 INFO:teuthology.lock.ops:Updating smithi175.front.sepia.ceph.com on lock server 2024-05-25T05:45:30.298 DEBUG:teuthology.orchestra.run.smithi179:> uname -m 2024-05-25T05:45:30.316 INFO:teuthology.orchestra.run.smithi179.stdout:x86_64 2024-05-25T05:45:30.316 DEBUG:teuthology.orchestra.run.smithi179:> cat /etc/os-release 2024-05-25T05:45:30.373 INFO:teuthology.orchestra.run.smithi179.stdout:NAME="CentOS Stream" 2024-05-25T05:45:30.373 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION="9" 2024-05-25T05:45:30.373 INFO:teuthology.orchestra.run.smithi179.stdout:ID="centos" 2024-05-25T05:45:30.373 INFO:teuthology.orchestra.run.smithi179.stdout:ID_LIKE="rhel fedora" 2024-05-25T05:45:30.373 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION_ID="9" 2024-05-25T05:45:30.373 INFO:teuthology.orchestra.run.smithi179.stdout:PLATFORM_ID="platform:el9" 2024-05-25T05:45:30.373 INFO:teuthology.orchestra.run.smithi179.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-25T05:45:30.374 INFO:teuthology.orchestra.run.smithi179.stdout:ANSI_COLOR="0;31" 2024-05-25T05:45:30.374 INFO:teuthology.orchestra.run.smithi179.stdout:LOGO="fedora-logo-icon" 2024-05-25T05:45:30.374 INFO:teuthology.orchestra.run.smithi179.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-25T05:45:30.374 INFO:teuthology.orchestra.run.smithi179.stdout:HOME_URL="https://centos.org/" 2024-05-25T05:45:30.374 INFO:teuthology.orchestra.run.smithi179.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-25T05:45:30.374 INFO:teuthology.orchestra.run.smithi179.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-25T05:45:30.374 INFO:teuthology.orchestra.run.smithi179.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-25T05:45:30.374 INFO:teuthology.lock.ops:Updating smithi179.front.sepia.ceph.com on lock server 2024-05-25T05:45:30.398 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-25T05:45:30.408 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-25T05:45:30.413 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-25T05:45:30.413 DEBUG:teuthology.orchestra.run.smithi175:> test '!' -e /home/ubuntu/cephtest 2024-05-25T05:45:30.415 DEBUG:teuthology.orchestra.run.smithi179:> test '!' -e /home/ubuntu/cephtest 2024-05-25T05:45:30.432 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-25T05:45:30.438 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-25T05:45:30.438 DEBUG:teuthology.orchestra.run.smithi175:> test -z $(ls -A /var/lib/ceph) 2024-05-25T05:45:30.472 DEBUG:teuthology.orchestra.run.smithi179:> test -z $(ls -A /var/lib/ceph) 2024-05-25T05:45:30.563 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-25T05:45:30.618 INFO:teuthology.run_tasks:Running task kernel... 2024-05-25T05:45:30.632 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-05-25T05:45:30.632 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-05-25T05:45:30.632 DEBUG:teuthology.orchestra.run.smithi175:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-25T05:45:30.633 DEBUG:teuthology.orchestra.run.smithi179:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-25T05:45:30.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T05:45:30.649 DEBUG:teuthology.orchestra.run.smithi175:> uname -r 2024-05-25T05:45:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T05:45:30.650 DEBUG:teuthology.orchestra.run.smithi179:> uname -r 2024-05-25T05:45:30.707 INFO:teuthology.orchestra.run.smithi175.stdout:5.14.0-437.el9.x86_64 2024-05-25T05:45:30.707 INFO:teuthology.task.kernel:Running kernel on smithi175: 5.14.0-437.el9.x86_64 2024-05-25T05:45:30.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum install -y kernel 2024-05-25T05:45:30.708 INFO:teuthology.orchestra.run.smithi179.stdout:5.14.0-437.el9.x86_64 2024-05-25T05:45:30.709 INFO:teuthology.task.kernel:Running kernel on smithi179: 5.14.0-437.el9.x86_64 2024-05-25T05:45:30.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum install -y kernel 2024-05-25T05:45:34.299 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - BaseOS 4.2 MB/s | 8.1 MB 00:01 2024-05-25T05:45:35.975 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - AppStream 32 MB/s | 19 MB 00:00 2024-05-25T05:45:36.518 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - BaseOS 1.9 MB/s | 8.1 MB 00:04 2024-05-25T05:45:39.852 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - AppStream 8.4 MB/s | 19 MB 00:02 2024-05-25T05:45:41.684 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - CRB 2.9 MB/s | 6.2 MB 00:02 2024-05-25T05:45:43.435 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - Extras packages 40 kB/s | 17 kB 00:00 2024-05-25T05:45:43.886 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.2 MB 00:00 2024-05-25T05:45:44.586 INFO:teuthology.orchestra.run.smithi179.stdout:Extra Packages for Enterprise Linux 21 MB/s | 21 MB 00:01 2024-05-25T05:45:45.337 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - Extras packages 55 kB/s | 17 kB 00:00 2024-05-25T05:45:46.196 INFO:teuthology.orchestra.run.smithi175.stdout:Extra Packages for Enterprise Linux 29 MB/s | 21 MB 00:00 2024-05-25T05:45:49.745 INFO:teuthology.orchestra.run.smithi179.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-05-25T05:45:51.413 INFO:teuthology.orchestra.run.smithi175.stdout:lab-extras 18 kB/s | 1.7 kB 00:00 2024-05-25T05:45:51.677 INFO:teuthology.orchestra.run.smithi179.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-25T05:45:51.677 INFO:teuthology.orchestra.run.smithi179.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-25T05:45:51.770 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-05-25T05:45:51.776 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-05-25T05:45:51.776 INFO:teuthology.orchestra.run.smithi179.stdout: Package Architecture Version Repository Size 2024-05-25T05:45:51.776 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout: kernel x86_64 5.14.0-447.el9 baseos 576 k 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-core x86_64 5.14.0-447.el9 baseos 16 M 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules x86_64 5.14.0-447.el9 baseos 34 M 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules-core x86_64 5.14.0-447.el9 baseos 28 M 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout:Install 4 Packages 2024-05-25T05:45:51.777 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:45:51.778 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 78 M 2024-05-25T05:45:51.778 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 123 M 2024-05-25T05:45:51.778 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-05-25T05:45:52.764 INFO:teuthology.orchestra.run.smithi179.stdout:(1/4): kernel-5.14.0-447.el9.x86_64.rpm 605 kB/s | 576 kB 00:00 2024-05-25T05:45:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-25T05:45:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-25T05:45:53.347 INFO:teuthology.orchestra.run.smithi179.stdout:(2/4): kernel-core-5.14.0-447.el9.x86_64.rpm 10 MB/s | 16 MB 00:01 2024-05-25T05:45:53.409 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout: Package Architecture Version Repository Size 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout: kernel x86_64 5.14.0-447.el9 baseos 576 k 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-core x86_64 5.14.0-447.el9 baseos 16 M 2024-05-25T05:45:53.416 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules x86_64 5.14.0-447.el9 baseos 34 M 2024-05-25T05:45:53.417 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-core x86_64 5.14.0-447.el9 baseos 28 M 2024-05-25T05:45:53.417 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:45:53.417 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-05-25T05:45:53.417 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-05-25T05:45:53.417 INFO:teuthology.orchestra.run.smithi175.stdout:Install 4 Packages 2024-05-25T05:45:53.417 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:45:53.417 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 78 M 2024-05-25T05:45:53.418 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 123 M 2024-05-25T05:45:53.418 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-05-25T05:45:53.989 INFO:teuthology.orchestra.run.smithi179.stdout:(3/4): kernel-modules-5.14.0-447.el9.x86_64.rpm 16 MB/s | 34 MB 00:02 2024-05-25T05:45:54.164 INFO:teuthology.orchestra.run.smithi175.stdout:(1/4): kernel-5.14.0-447.el9.x86_64.rpm 1.2 MB/s | 576 kB 00:00 2024-05-25T05:45:55.131 INFO:teuthology.orchestra.run.smithi179.stdout:(4/4): kernel-modules-core-5.14.0-447.el9.x86_6 12 MB/s | 28 MB 00:02 2024-05-25T05:45:55.131 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-05-25T05:45:55.132 INFO:teuthology.orchestra.run.smithi179.stdout:Total 23 MB/s | 78 MB 00:03 2024-05-25T05:45:55.818 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-05-25T05:45:56.155 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-05-25T05:45:56.155 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-05-25T05:45:56.764 INFO:teuthology.orchestra.run.smithi175.stdout:(2/4): kernel-core-5.14.0-447.el9.x86_64.rpm 5.0 MB/s | 16 MB 00:03 2024-05-25T05:45:57.068 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-05-25T05:45:57.068 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-05-25T05:45:59.459 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-05-25T05:46:00.146 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-modules-core-5.14.0-447.el9.x86_64 1/4 2024-05-25T05:46:00.310 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-25T05:46:01.121 INFO:teuthology.orchestra.run.smithi175.stdout:(3/4): kernel-modules-5.14.0-447.el9.x86_64.rpm 4.6 MB/s | 34 MB 00:07 2024-05-25T05:46:01.380 INFO:teuthology.orchestra.run.smithi175.stdout:(4/4): kernel-modules-core-5.14.0-447.el9.x86_6 3.9 MB/s | 28 MB 00:07 2024-05-25T05:46:01.380 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-05-25T05:46:01.380 INFO:teuthology.orchestra.run.smithi175.stdout:Total 9.8 MB/s | 78 MB 00:07 2024-05-25T05:46:02.069 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-05-25T05:46:02.400 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-05-25T05:46:02.400 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-05-25T05:46:02.959 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-25T05:46:03.109 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-25T05:46:03.350 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-05-25T05:46:03.351 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-05-25T05:46:05.719 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-05-25T05:46:06.392 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-modules-core-5.14.0-447.el9.x86_64 1/4 2024-05-25T05:46:06.505 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-25T05:46:07.809 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-25T05:46:07.855 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:09.151 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-25T05:46:09.296 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-25T05:46:12.570 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:14.022 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-25T05:46:14.065 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:18.779 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:42.305 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:42.344 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:42.680 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:42.680 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-5.14.0-447.el9.x86_64 1/4 2024-05-25T05:46:42.681 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-25T05:46:42.681 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-5.14.0-447.el9.x86_64 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-core-5.14.0-447.el9.x86_64 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules-5.14.0-447.el9.x86_64 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules-core-5.14.0-447.el9.x86_64 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:46:43.526 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-05-25T05:46:43.923 DEBUG:teuthology.orchestra.run.smithi179:> rpm -q kernel | sort -rV | head -n 1 2024-05-25T05:46:44.001 INFO:teuthology.orchestra.run.smithi179.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-25T05:46:44.001 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-25T05:46:44.001 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-447.el9.x86_64 2024-05-25T05:46:44.001 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-25T05:46:44.002 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-25T05:46:44.002 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-05-25T05:46:44.002 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi179.front.sepia.ceph.com, path=None, version=distro) 2024-05-25T05:46:44.002 DEBUG:teuthology.orchestra.run.smithi179:> rpm -q kernel | sort -rV | head -n 1 2024-05-25T05:46:44.052 INFO:teuthology.orchestra.run.smithi179.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-25T05:46:44.052 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-25T05:46:44.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo rpm -qi grub2-tools 2024-05-25T05:46:44.082 INFO:teuthology.orchestra.run.smithi179.stdout:Name : grub2-tools 2024-05-25T05:46:44.082 INFO:teuthology.orchestra.run.smithi179.stdout:Epoch : 1 2024-05-25T05:46:44.082 INFO:teuthology.orchestra.run.smithi179.stdout:Version : 2.06 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Release : 68.el9 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Architecture: x86_64 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Group : Unspecified 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Size : 8264254 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:License : GPLv3+ 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Packager : builder@centos.org 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Vendor : CentOS 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-25T05:46:44.083 INFO:teuthology.orchestra.run.smithi179.stdout:Summary : Support tools for GRUB. 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout:Description : 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout:hardware devices. 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:46:44.084 INFO:teuthology.orchestra.run.smithi179.stdout:This subpackage provides tools for support of all platforms. 2024-05-25T05:46:44.085 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-25T05:46:44.085 INFO:teuthology.task.kernel:Updating grub on smithi179 to boot 5.14.0-447.el9.x86_64 2024-05-25T05:46:44.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-25T05:46:44.687 INFO:teuthology.orchestra.run.smithi179.stderr:Generating grub configuration file ... 2024-05-25T05:46:46.114 INFO:teuthology.orchestra.run.smithi179.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-25T05:46:46.136 INFO:teuthology.orchestra.run.smithi179.stderr:done 2024-05-25T05:46:46.138 DEBUG:teuthology.orchestra.run.smithi179:> mktemp 2024-05-25T05:46:46.153 INFO:teuthology.orchestra.run.smithi179.stdout:/tmp/tmp.KfqXI3tBJf 2024-05-25T05:46:46.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.KfqXI3tBJf 2024-05-25T05:46:46.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod 0666 /tmp/tmp.KfqXI3tBJf 2024-05-25T05:46:46.360 DEBUG:teuthology.orchestra.remote:smithi179:/tmp/tmp.KfqXI3tBJf is 6KB 2024-05-25T05:46:46.410 DEBUG:teuthology.orchestra.run.smithi179:> rm -fr /tmp/tmp.KfqXI3tBJf 2024-05-25T05:46:46.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo /bin/ls /boot/loader/entries || true 2024-05-25T05:46:46.485 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-25T05:46:46.485 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-25T05:46:46.485 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-25T05:46:46.485 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64.conf 2024-05-25T05:46:46.486 DEBUG:teuthology.orchestra.run.smithi179:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64 2024-05-25T05:46:46.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo shutdown -r now 2024-05-25T05:46:47.854 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:47.893 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:48.182 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:48.183 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-5.14.0-447.el9.x86_64 1/4 2024-05-25T05:46:48.183 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-25T05:46:48.183 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-25T05:46:48.970 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-5.14.0-447.el9.x86_64 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-core-5.14.0-447.el9.x86_64 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-5.14.0-447.el9.x86_64 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-core-5.14.0-447.el9.x86_64 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:46:48.971 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-05-25T05:46:49.333 DEBUG:teuthology.orchestra.run.smithi175:> rpm -q kernel | sort -rV | head -n 1 2024-05-25T05:46:49.420 INFO:teuthology.orchestra.run.smithi175.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-25T05:46:49.421 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-25T05:46:49.421 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-447.el9.x86_64 2024-05-25T05:46:49.421 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-25T05:46:49.421 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-25T05:46:49.421 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-05-25T05:46:49.421 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi175.front.sepia.ceph.com, path=None, version=distro) 2024-05-25T05:46:49.421 DEBUG:teuthology.orchestra.run.smithi175:> rpm -q kernel | sort -rV | head -n 1 2024-05-25T05:46:49.470 INFO:teuthology.orchestra.run.smithi175.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-25T05:46:49.471 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-25T05:46:49.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo rpm -qi grub2-tools 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Name : grub2-tools 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Epoch : 1 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Version : 2.06 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Release : 68.el9 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Architecture: x86_64 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Group : Unspecified 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Size : 8264254 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:License : GPLv3+ 2024-05-25T05:46:49.501 INFO:teuthology.orchestra.run.smithi175.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:Packager : builder@centos.org 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:Vendor : CentOS 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:Summary : Support tools for GRUB. 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:Description : 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout:hardware devices. 2024-05-25T05:46:49.502 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:46:49.503 INFO:teuthology.orchestra.run.smithi175.stdout:This subpackage provides tools for support of all platforms. 2024-05-25T05:46:49.503 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-25T05:46:49.503 INFO:teuthology.task.kernel:Updating grub on smithi175 to boot 5.14.0-447.el9.x86_64 2024-05-25T05:46:49.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-25T05:46:50.127 INFO:teuthology.orchestra.run.smithi175.stderr:Generating grub configuration file ... 2024-05-25T05:46:51.511 INFO:teuthology.orchestra.run.smithi175.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-25T05:46:51.542 INFO:teuthology.orchestra.run.smithi175.stderr:done 2024-05-25T05:46:51.545 DEBUG:teuthology.orchestra.run.smithi175:> mktemp 2024-05-25T05:46:51.563 INFO:teuthology.orchestra.run.smithi175.stdout:/tmp/tmp.QbOos6jTft 2024-05-25T05:46:51.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.QbOos6jTft 2024-05-25T05:46:51.636 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 0666 /tmp/tmp.QbOos6jTft 2024-05-25T05:46:51.771 DEBUG:teuthology.orchestra.remote:smithi175:/tmp/tmp.QbOos6jTft is 6KB 2024-05-25T05:46:51.823 DEBUG:teuthology.orchestra.run.smithi175:> rm -fr /tmp/tmp.QbOos6jTft 2024-05-25T05:46:51.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo /bin/ls /boot/loader/entries || true 2024-05-25T05:46:51.900 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-25T05:46:51.900 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-25T05:46:51.900 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-25T05:46:51.900 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64.conf 2024-05-25T05:46:51.901 DEBUG:teuthology.orchestra.run.smithi175:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64 2024-05-25T05:46:52.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo shutdown -r now 2024-05-25T05:47:16.666 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-05-25T05:47:16.667 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-05-25T05:47:16.667 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:47:22.047 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-05-25T05:47:22.047 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-05-25T05:47:22.048 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:48:16.707 DEBUG:teuthology.orchestra.remote:timed out 2024-05-25T05:48:22.050 DEBUG:teuthology.orchestra.remote:timed out 2024-05-25T05:48:25.712 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-05-25T05:48:25.713 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:48:29.258 DEBUG:teuthology.orchestra.run.smithi179:> true 2024-05-25T05:48:29.656 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-05-25T05:48:29.656 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-447.el9.x86_64"... 2024-05-25T05:48:29.656 DEBUG:teuthology.orchestra.run.smithi179:> uname -r 2024-05-25T05:48:29.671 INFO:teuthology.orchestra.run.smithi179.stdout:5.14.0-447.el9.x86_64 2024-05-25T05:48:29.671 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-447.el9.x86_64 vs 5.14.0-447.el9.x86_64 2024-05-25T05:48:29.671 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-25T05:48:29.671 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-25T05:48:30.672 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-05-25T05:48:30.672 DEBUG:teuthology.orchestra.run.smithi179:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-25T05:48:30.768 INFO:teuthology.orchestra.run.smithi179.stdout:ttyS1 2024-05-25T05:48:30.804 DEBUG:teuthology.parallel:result is None 2024-05-25T05:48:31.051 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-05-25T05:48:31.051 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:48:32.328 DEBUG:teuthology.orchestra.run.smithi175:> true 2024-05-25T05:48:32.826 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-05-25T05:48:32.826 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-447.el9.x86_64"... 2024-05-25T05:48:32.827 DEBUG:teuthology.orchestra.run.smithi175:> uname -r 2024-05-25T05:48:32.841 INFO:teuthology.orchestra.run.smithi175.stdout:5.14.0-447.el9.x86_64 2024-05-25T05:48:32.841 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-447.el9.x86_64 vs 5.14.0-447.el9.x86_64 2024-05-25T05:48:32.841 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-25T05:48:32.841 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-25T05:48:33.842 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-05-25T05:48:33.843 DEBUG:teuthology.orchestra.run.smithi175:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-25T05:48:33.917 INFO:teuthology.orchestra.run.smithi175.stdout:ttyS1 2024-05-25T05:48:33.950 DEBUG:teuthology.parallel:result is None 2024-05-25T05:48:33.950 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-25T05:48:33.957 INFO:teuthology.task.internal:Creating test directory... 2024-05-25T05:48:33.957 DEBUG:teuthology.orchestra.run.smithi175:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-25T05:48:33.959 DEBUG:teuthology.orchestra.run.smithi179:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-25T05:48:33.975 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-25T05:48:33.981 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-25T05:48:33.988 INFO:teuthology.task.internal:Creating archive directory... 2024-05-25T05:48:33.989 DEBUG:teuthology.orchestra.run.smithi175:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-25T05:48:34.015 DEBUG:teuthology.orchestra.run.smithi179:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-25T05:48:34.041 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-25T05:48:34.047 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-25T05:48:34.047 DEBUG:teuthology.orchestra.run.smithi175:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-25T05:48:34.080 DEBUG:teuthology.orchestra.run.smithi179:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-25T05:48:34.111 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-25T05:48:34.117 INFO:teuthology.orchestra.run.smithi179.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-25T05:48:34.121 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-25T05:48:34.127 INFO:teuthology.orchestra.run.smithi179.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-25T05:48:34.129 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-25T05:48:34.137 INFO:teuthology.task.internal:Configuring sudo... 2024-05-25T05:48:34.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-25T05:48:34.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-25T05:48:34.193 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-25T05:48:34.203 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-25T05:48:34.203 DEBUG:teuthology.orchestra.run.smithi175:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-25T05:48:34.231 DEBUG:teuthology.orchestra.run.smithi179:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-25T05:48:34.250 DEBUG:teuthology.orchestra.run.smithi175:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-25T05:48:34.327 DEBUG:teuthology.orchestra.run.smithi175:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-25T05:48:34.376 DEBUG:teuthology.orchestra.run.smithi175:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-25T05:48:34.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-25T05:48:34.497 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:48:34.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-25T05:48:34.580 DEBUG:teuthology.orchestra.run.smithi179:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-25T05:48:34.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-25T05:48:34.704 DEBUG:teuthology.orchestra.run.smithi179:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-25T05:48:34.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-25T05:48:34.785 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-05-25T05:48:34.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-25T05:48:34.850 DEBUG:teuthology.orchestra.run.smithi175:> sudo service rsyslog restart 2024-05-25T05:48:34.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo service rsyslog restart 2024-05-25T05:48:34.885 INFO:teuthology.orchestra.run.smithi175.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-25T05:48:35.130 INFO:teuthology.orchestra.run.smithi179.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-25T05:48:35.618 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-25T05:48:35.627 INFO:teuthology.task.internal:Starting timer... 2024-05-25T05:48:35.627 INFO:teuthology.run_tasks:Running task pcp... 2024-05-25T05:48:35.637 INFO:teuthology.run_tasks:Running task selinux... 2024-05-25T05:48:35.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo service auditd rotate 2024-05-25T05:48:35.715 INFO:teuthology.orchestra.run.smithi175.stdout:Rotating logs: 2024-05-25T05:48:35.717 DEBUG:teuthology.orchestra.run.smithi179:> sudo service auditd rotate 2024-05-25T05:48:35.804 INFO:teuthology.orchestra.run.smithi179.stdout:Rotating logs: 2024-05-25T05:48:35.807 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-25T05:48:35.808 DEBUG:teuthology.orchestra.run.smithi175:> /usr/sbin/getenforce 2024-05-25T05:48:35.832 INFO:teuthology.orchestra.run.smithi175.stdout:Permissive 2024-05-25T05:48:35.832 DEBUG:teuthology.orchestra.run.smithi179:> /usr/sbin/getenforce 2024-05-25T05:48:35.869 INFO:teuthology.orchestra.run.smithi179.stdout:Permissive 2024-05-25T05:48:35.869 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi175.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi179.front.sepia.ceph.com': 'permissive'} 2024-05-25T05:48:35.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-25T05:48:35.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T05:48:35.897 DEBUG:teuthology.orchestra.run.smithi179:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-25T05:48:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T05:48:35.956 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-25T05:48:35.956 DEBUG:teuthology.orchestra.run.smithi175:> sudo /usr/sbin/setenforce permissive 2024-05-25T05:48:35.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo /usr/sbin/setenforce permissive 2024-05-25T05:48:36.023 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-25T05:48:36.033 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-05-25T05:48:36.036 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-05-25T05:48:36.149 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-25T05:48:36.159 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-25T05:48:36.160 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi175.front.sepia.ceph.com,smithi179.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-25T05:53:01.076 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi175.front.sepia.ceph.com'), Remote(name='ubuntu@smithi179.front.sepia.ceph.com')] 2024-05-25T05:53:01.078 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-05-25T05:53:01.079 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:53:01.158 DEBUG:teuthology.orchestra.run.smithi175:> true 2024-05-25T05:53:01.237 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-05-25T05:53:01.238 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-05-25T05:53:01.238 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-25T05:53:01.313 DEBUG:teuthology.orchestra.run.smithi179:> true 2024-05-25T05:53:01.393 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-05-25T05:53:01.394 INFO:teuthology.run_tasks:Running task clock... 2024-05-25T05:53:01.407 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-25T05:53:01.408 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-25T05:53:01.408 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-25T05:53:01.411 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-25T05:53:01.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-25T05:53:01.438 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-25T05:53:01.452 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-25T05:53:01.462 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-25T05:53:01.476 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-25T05:53:01.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: ntpd: command not found 2024-05-25T05:53:01.487 INFO:teuthology.orchestra.run.smithi175.stdout:506 Cannot talk to daemon 2024-05-25T05:53:01.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: ntpd: command not found 2024-05-25T05:53:01.501 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-25T05:53:01.511 INFO:teuthology.orchestra.run.smithi179.stdout:506 Cannot talk to daemon 2024-05-25T05:53:01.515 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-25T05:53:01.525 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-25T05:53:01.539 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-25T05:53:01.573 INFO:teuthology.orchestra.run.smithi175.stderr:bash: line 1: ntpq: command not found 2024-05-25T05:53:01.577 INFO:teuthology.orchestra.run.smithi175.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-25T05:53:01.577 INFO:teuthology.orchestra.run.smithi175.stdout:=============================================================================== 2024-05-25T05:53:01.577 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.577 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.577 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.577 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.595 INFO:teuthology.orchestra.run.smithi179.stderr:bash: line 1: ntpq: command not found 2024-05-25T05:53:01.599 INFO:teuthology.orchestra.run.smithi179.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-25T05:53:01.599 INFO:teuthology.orchestra.run.smithi179.stdout:=============================================================================== 2024-05-25T05:53:01.599 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.599 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.599 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.599 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T05:53:01.602 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-25T05:53:01.708 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'ea0684c08cbcf83f4af9930de2267b3563ac5f78'} 2024-05-25T05:53:01.708 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-05-25T05:53:01.708 INFO:tasks.cephadm:Cluster fsid is 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:01.708 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-25T05:53:01.709 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.175', 'mon.c': '[v2:172.21.15.175:3301,v1:172.21.15.175:6790]', 'mon.b': '172.21.15.179'} 2024-05-25T05:53:01.709 INFO:tasks.cephadm:First mon is mon.a on smithi175 2024-05-25T05:53:01.709 INFO:tasks.cephadm:First mgr is y 2024-05-25T05:53:01.709 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-25T05:53:01.709 DEBUG:teuthology.orchestra.run.smithi175:> sudo hostname $(hostname -s) 2024-05-25T05:53:01.735 DEBUG:teuthology.orchestra.run.smithi179:> sudo hostname $(hostname -s) 2024-05-25T05:53:01.762 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-05-25T05:53:01.763 DEBUG:teuthology.orchestra.run.smithi175:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-25T05:53:02.049 INFO:teuthology.orchestra.run.smithi175.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 25 05:53 /home/ubuntu/cephtest/cephadm 2024-05-25T05:53:02.050 DEBUG:teuthology.orchestra.run.smithi179:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-25T05:53:02.292 INFO:teuthology.orchestra.run.smithi179.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 25 05:53 /home/ubuntu/cephtest/cephadm 2024-05-25T05:53:02.293 DEBUG:teuthology.orchestra.run.smithi175:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-25T05:53:02.309 DEBUG:teuthology.orchestra.run.smithi179:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-25T05:53:02.339 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-05-25T05:53:02.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-25T05:53:02.351 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-25T05:53:03.199 INFO:teuthology.orchestra.run.smithi179.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-25T05:53:03.216 INFO:teuthology.orchestra.run.smithi175.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-25T05:53:25.407 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-05-25T05:53:25.408 INFO:teuthology.orchestra.run.smithi179.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-25T05:53:25.408 INFO:teuthology.orchestra.run.smithi179.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-25T05:53:25.408 INFO:teuthology.orchestra.run.smithi179.stdout: "repo_digests": [ 2024-05-25T05:53:25.408 INFO:teuthology.orchestra.run.smithi179.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-25T05:53:25.408 INFO:teuthology.orchestra.run.smithi179.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-25T05:53:25.409 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-05-25T05:53:25.409 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-05-25T05:53:25.944 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T05:53:25.945 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-25T05:53:25.945 INFO:teuthology.orchestra.run.smithi175.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-25T05:53:25.945 INFO:teuthology.orchestra.run.smithi175.stdout: "repo_digests": [ 2024-05-25T05:53:25.945 INFO:teuthology.orchestra.run.smithi175.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-25T05:53:25.945 INFO:teuthology.orchestra.run.smithi175.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-25T05:53:25.945 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-05-25T05:53:25.945 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T05:53:25.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo mkdir -p /etc/ceph 2024-05-25T05:53:26.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo mkdir -p /etc/ceph 2024-05-25T05:53:26.054 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 777 /etc/ceph 2024-05-25T05:53:26.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod 777 /etc/ceph 2024-05-25T05:53:26.134 INFO:tasks.cephadm:Writing seed config... 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-25T05:53:26.136 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-25T05:53:26.137 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:53:26.137 DEBUG:teuthology.orchestra.run.smithi175:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-25T05:53:26.166 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-05-25T05:53:26.166 DEBUG:teuthology.orchestra.run.smithi175:mon.a> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service 2024-05-25T05:53:26.209 DEBUG:teuthology.orchestra.run.smithi175:mgr.y> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service 2024-05-25T05:53:26.252 INFO:tasks.cephadm:Bootstrapping... 2024-05-25T05:53:26.252 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.175 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-25T05:53:26.444 INFO:teuthology.orchestra.run.smithi175.stderr:-------------------------------------------------------------------------------- 2024-05-25T05:53:26.444 INFO:teuthology.orchestra.run.smithi175.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '0bfa7ec4-1a5b-11ef-bc9b-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.175', '--skip-admin-label'] 2024-05-25T05:53:26.468 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: 5.0.2 2024-05-25T05:53:26.470 INFO:teuthology.orchestra.run.smithi175.stderr:Verifying podman|docker is present... 2024-05-25T05:53:26.491 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: 5.0.2 2024-05-25T05:53:26.493 INFO:teuthology.orchestra.run.smithi175.stderr:Verifying lvm2 is present... 2024-05-25T05:53:26.493 INFO:teuthology.orchestra.run.smithi175.stderr:Verifying time synchronization is in place... 2024-05-25T05:53:26.501 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-25T05:53:26.507 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: inactive 2024-05-25T05:53:26.514 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: enabled 2024-05-25T05:53:26.520 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: active 2024-05-25T05:53:26.521 INFO:teuthology.orchestra.run.smithi175.stderr:Unit chronyd.service is enabled and running 2024-05-25T05:53:26.521 INFO:teuthology.orchestra.run.smithi175.stderr:Repeating the final host check... 2024-05-25T05:53:26.541 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: 5.0.2 2024-05-25T05:53:26.543 INFO:teuthology.orchestra.run.smithi175.stderr:podman (/bin/podman) version 5.0.2 is present 2024-05-25T05:53:26.543 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl is present 2024-05-25T05:53:26.543 INFO:teuthology.orchestra.run.smithi175.stderr:lvcreate is present 2024-05-25T05:53:26.549 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-25T05:53:26.556 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: inactive 2024-05-25T05:53:26.563 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: enabled 2024-05-25T05:53:26.569 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: active 2024-05-25T05:53:26.570 INFO:teuthology.orchestra.run.smithi175.stderr:Unit chronyd.service is enabled and running 2024-05-25T05:53:26.570 INFO:teuthology.orchestra.run.smithi175.stderr:Host looks OK 2024-05-25T05:53:26.570 INFO:teuthology.orchestra.run.smithi175.stderr:Cluster fsid: 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:26.570 INFO:teuthology.orchestra.run.smithi175.stderr:Acquiring lock 140229685388288 on /run/cephadm/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.lock 2024-05-25T05:53:26.570 INFO:teuthology.orchestra.run.smithi175.stderr:Lock 140229685388288 acquired on /run/cephadm/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.lock 2024-05-25T05:53:26.571 INFO:teuthology.orchestra.run.smithi175.stderr:Verifying IP 172.21.15.175 port 3300 ... 2024-05-25T05:53:26.571 INFO:teuthology.orchestra.run.smithi175.stderr:Verifying IP 172.21.15.175 port 6789 ... 2024-05-25T05:53:26.571 INFO:teuthology.orchestra.run.smithi175.stderr:Base mon IP is 172.21.15.175, final addrv is [v2:172.21.15.175:3300,v1:172.21.15.175:6789] 2024-05-25T05:53:26.573 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.175 metric 100 2024-05-25T05:53:26.573 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.175 metric 100 2024-05-25T05:53:26.576 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-05-25T05:53:26.576 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-05-25T05:53:26.576 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1795sec hoplimit 64 pref medium 2024-05-25T05:53:26.579 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-25T05:53:26.579 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-05-25T05:53:26.579 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-25T05:53:26.579 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-05-25T05:53:26.579 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:71f1/64 scope link 2024-05-25T05:53:26.579 INFO:teuthology.orchestra.run.smithi175.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-25T05:53:26.580 INFO:teuthology.orchestra.run.smithi175.stderr:Mon IP `172.21.15.175` is in CIDR network `172.21.0.0/20` 2024-05-25T05:53:26.580 INFO:teuthology.orchestra.run.smithi175.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-25T05:53:26.580 INFO:teuthology.orchestra.run.smithi175.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-25T05:53:26.681 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-05-25T05:53:27.119 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Getting image source signatures 2024-05-25T05:53:27.120 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-05-25T05:53:27.120 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-05-25T05:53:27.120 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-05-25T05:53:27.120 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-05-25T05:53:27.120 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-05-25T05:53:27.120 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-25T05:53:27.256 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: Writing manifest to image destination 2024-05-25T05:53:27.408 INFO:teuthology.orchestra.run.smithi175.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-25T05:53:27.786 INFO:teuthology.orchestra.run.smithi175.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-25T05:53:28.282 INFO:teuthology.orchestra.run.smithi175.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-25T05:53:28.283 INFO:teuthology.orchestra.run.smithi175.stderr:Extracting ceph user uid/gid from container image... 2024-05-25T05:53:28.759 INFO:teuthology.orchestra.run.smithi175.stderr:stat: 167 167 2024-05-25T05:53:29.339 INFO:teuthology.orchestra.run.smithi175.stderr:Creating initial keys... 2024-05-25T05:53:29.714 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph-authtool: AQDZfFFm4CyFJRAAT7L6mFbNxKtB4UbQB9JCbQ== 2024-05-25T05:53:30.595 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph-authtool: AQDafFFmqFnvHhAAH91y5hdUpeQnDX2jppPqMg== 2024-05-25T05:53:31.543 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph-authtool: AQDbfFFmY79WGxAAYRCeCBsWVCpcK2hpXblspw== 2024-05-25T05:53:32.184 INFO:teuthology.orchestra.run.smithi175.stderr:Creating initial monmap... 2024-05-25T05:53:32.599 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-25T05:53:32.599 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-05-25T05:53:32.599 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:32.599 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-25T05:53:33.047 INFO:teuthology.orchestra.run.smithi175.stderr:monmaptool for a [v2:172.21.15.175:3300,v1:172.21.15.175:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-25T05:53:33.048 INFO:teuthology.orchestra.run.smithi175.stderr:setting min_mon_release = octopus 2024-05-25T05:53:33.048 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/monmaptool: set fsid to 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:33.048 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-25T05:53:33.048 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:53:33.048 INFO:teuthology.orchestra.run.smithi175.stderr:Creating mon... 2024-05-25T05:53:33.936 INFO:teuthology.orchestra.run.smithi175.stderr:create mon.a on 2024-05-25T05:53:34.132 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-25T05:53:34.274 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.target → /etc/systemd/system/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.target. 2024-05-25T05:53:34.274 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.target → /etc/systemd/system/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.target. 2024-05-25T05:53:34.569 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Failed to reset failed state of unit ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service: Unit ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service not loaded. 2024-05-25T05:53:34.576 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Created symlink /etc/systemd/system/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.target.wants/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service → /etc/systemd/system/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@.service. 2024-05-25T05:53:35.247 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:35 smithi175 ceph-mon[24442]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-25T05:53:35.247 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:35 smithi175 bash[24424]: 1e53daeca1edf745f031ce3b2f6ba5c38ebc73b3763346f291b7c52b26f9f930 2024-05-25T05:53:35.259 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: disabled 2024-05-25T05:53:35.266 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: inactive 2024-05-25T05:53:35.267 INFO:teuthology.orchestra.run.smithi175.stderr:firewalld.service is not enabled 2024-05-25T05:53:35.267 INFO:teuthology.orchestra.run.smithi175.stderr:Not possible to enable service . firewalld.service is not available 2024-05-25T05:53:35.267 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for mon to start... 2024-05-25T05:53:35.267 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for mon... 2024-05-25T05:53:35.513 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:35 smithi175 systemd[1]: Started Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: cluster: 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: id: 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: services: 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.57545s) 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mgr: no daemons active 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:35.764 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: data: 2024-05-25T05:53:35.765 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-05-25T05:53:35.765 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-05-25T05:53:35.765 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-05-25T05:53:35.765 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: pgs: 2024-05-25T05:53:35.765 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:36.239 INFO:teuthology.orchestra.run.smithi175.stderr:mon is available 2024-05-25T05:53:36.240 INFO:teuthology.orchestra.run.smithi175.stderr:Assimilating anything we can from ceph.conf... 2024-05-25T05:53:36.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:36 smithi175 ceph-mon[24442]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-25T05:53:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:36 smithi175 ceph-mon[24442]: monmap e1: 1 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0]} 2024-05-25T05:53:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:36 smithi175 ceph-mon[24442]: fsmap 2024-05-25T05:53:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:36 smithi175 ceph-mon[24442]: osdmap e1: 0 total, 0 up, 0 in 2024-05-25T05:53:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:36 smithi175 ceph-mon[24442]: mgrmap e1: no daemons active 2024-05-25T05:53:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:36 smithi175 ceph-mon[24442]: from='client.? 172.21.15.175:0/2835515725' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: [global] 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: fsid = 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.175:3300,v1:172.21.15.175:6789] 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: [mgr] 2024-05-25T05:53:36.673 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-05-25T05:53:36.674 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-25T05:53:36.674 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:36.674 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: [osd] 2024-05-25T05:53:36.674 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-25T05:53:36.674 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-25T05:53:37.171 INFO:teuthology.orchestra.run.smithi175.stderr:Generating new minimal ceph.conf... 2024-05-25T05:53:37.269 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:37 smithi175 ceph-mon[24442]: from='client.? 172.21.15.175:0/624444209' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-25T05:53:37.269 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:37 smithi175 ceph-mon[24442]: from='client.? 172.21.15.175:0/624444209' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-25T05:53:38.136 INFO:teuthology.orchestra.run.smithi175.stderr:Restarting the monitor... 2024-05-25T05:53:38.190 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 systemd[1]: Stopping Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T05:53:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 bash[24658]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon.a" found: no such container 2024-05-25T05:53:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 ceph-mon[24442]: from='client.? 172.21.15.175:0/2583637165' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:53:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a[24438]: 2024-05-25T05:53:38.304+0000 7f15cd6c4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T05:53:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a[24438]: 2024-05-25T05:53:38.304+0000 7f15cd6c4700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-05-25T05:53:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 podman[24669]: 2024-05-25 05:53:38.365670189 +0000 UTC m=+0.165698377 container died 1e53daeca1edf745f031ce3b2f6ba5c38ebc73b3763346f291b7c52b26f9f930 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vcs-type=git, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., architecture=x86_64, RELEASE=HEAD) 2024-05-25T05:53:38.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 podman[24669]: 2024-05-25 05:53:38.678370772 +0000 UTC m=+0.478398961 container cleanup 1e53daeca1edf745f031ce3b2f6ba5c38ebc73b3763346f291b7c52b26f9f930 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, io.k8s.display-name=CentOS Stream 8, ceph=True, architecture=x86_64, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.buildah.version=1.19.8) 2024-05-25T05:53:38.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 bash[24669]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a 2024-05-25T05:53:38.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 bash[24694]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon.a" found: no such container 2024-05-25T05:53:38.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 podman[24681]: 2024-05-25 05:53:38.857814808 +0000 UTC m=+0.489693412 container remove 1e53daeca1edf745f031ce3b2f6ba5c38ebc73b3763346f291b7c52b26f9f930 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, release=754, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vcs-type=git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., distribution-scope=public) 2024-05-25T05:53:38.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-25T05:53:38.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 systemd[1]: Stopped Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:53:39.190 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:38 smithi175 systemd[1]: Starting Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T05:53:39.191 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 podman[24758]: 2024-05-25 05:53:39.085419866 +0000 UTC m=+0.113124155 container create b34a4a648ba02b1e6c165033a6f89f34f239deb13bf69d4277c4adfc977ad085 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, name=centos-stream, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-25T05:53:39.191 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 podman[24758]: 2024-05-25 05:53:38.99018761 +0000 UTC m=+0.017891881 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-25T05:53:39.323 INFO:teuthology.orchestra.run.smithi175.stderr:Setting mon public_network to 172.21.0.0/20 2024-05-25T05:53:39.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 podman[24758]: 2024-05-25 05:53:39.262559363 +0000 UTC m=+0.290263629 container init b34a4a648ba02b1e6c165033a6f89f34f239deb13bf69d4277c4adfc977ad085 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, version=8, architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-25T05:53:39.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 podman[24758]: 2024-05-25 05:53:39.265524758 +0000 UTC m=+0.293229025 container start b34a4a648ba02b1e6c165033a6f89f34f239deb13bf69d4277c4adfc977ad085 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, distribution-scope=public, io.openshift.expose-services=, version=8, GIT_BRANCH=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8) 2024-05-25T05:53:39.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: set uid:gid to 167:167 (ceph:ceph) 2024-05-25T05:53:39.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-25T05:53:39.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: pidfile_write: ignore empty --pid-file 2024-05-25T05:53:39.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: load: jerasure load: lrc 2024-05-25T05:53:39.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: RocksDB version: 6.15.5 2024-05-25T05:53:39.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Compile date Apr 18 2022 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: DB SUMMARY 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: DB Session ID: L88TTV4ND4DIJGI43KMC 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: CURRENT file: CURRENT 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: IDENTITY file: IDENTITY 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82655 ; 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.error_if_exists: 0 2024-05-25T05:53:39.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.create_if_missing: 0 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.paranoid_checks: 1 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.env: 0x55f41bec6860 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.fs: Posix File System 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.info_log: 0x55f41c979dc0 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_file_opening_threads: 16 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.statistics: (nil) 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.use_fsync: 0 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_log_file_size: 0 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-25T05:53:39.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.keep_log_file_num: 1000 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.recycle_log_file_num: 0 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.allow_fallocate: 1 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.allow_mmap_reads: 0 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.allow_mmap_writes: 0 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.use_direct_reads: 0 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.create_missing_column_families: 0 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.db_log_dir: 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-25T05:53:39.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.advise_random_on_open: 1 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.db_write_buffer_size: 0 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.write_buffer_manager: 0x55f41ca6a240 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.rate_limiter: (nil) 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-25T05:53:39.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.wal_recovery_mode: 2 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.enable_thread_tracking: 0 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.enable_pipelined_write: 0 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.unordered_write: 0 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.row_cache: None 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.wal_filter: None 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.allow_ingest_behind: 0 2024-05-25T05:53:39.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.preserve_deletes: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.two_write_queues: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.manual_wal_flush: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.atomic_flush: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.log_readahead_size: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.best_efforts_recovery: 0 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-25T05:53:39.577 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.allow_data_in_errors: 0 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.db_host_id: __hostname__ 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_background_jobs: 2 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_background_compactions: -1 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_subcompactions: 1 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_total_wal_size: 0 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-25T05:53:39.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_open_files: -1 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bytes_per_sync: 0 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_readahead_size: 0 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_background_flushes: -1 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Compression algorithms supported: 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-25T05:53:39.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kZSTD supported: 0 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kXpressCompression supported: 0 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kLZ4HCCompression supported: 1 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kLZ4Compression supported: 1 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kBZip2Compression supported: 0 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kZlibCompression supported: 1 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: kSnappyCompression supported: 1 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.merge_operator: 2024-05-25T05:53:39.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_filter: None 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_filter_factory: None 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.sst_partitioner_factory: None 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f41c945d00) 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: cache_index_and_filter_blocks: 1 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: pin_top_level_index_and_filter: 1 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: index_type: 0 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: data_block_index_type: 0 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: index_shortening: 1 2024-05-25T05:53:39.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: hash_index_allow_collision: 1 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: checksum: 1 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: no_block_cache: 0 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache: 0x55f41c9b0170 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache_name: BinnedLRUCache 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache_options: 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: capacity : 536870912 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: num_shard_bits : 4 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: strict_capacity_limit : 0 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: high_pri_pool_ratio: 0.000 2024-05-25T05:53:39.582 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache_compressed: (nil) 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: persistent_cache: (nil) 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_size: 4096 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_size_deviation: 10 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_restart_interval: 16 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: index_block_restart_interval: 1 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: metadata_block_size: 4096 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: partition_filters: 0 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_delta_encoding: 1 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: whole_key_filtering: 1 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: verify_compression: 0 2024-05-25T05:53:39.583 INFO:journalctl@ceph.mon.a.smithi175.stdout: read_amp_bytes_per_bit: 0 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout: format_version: 4 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout: enable_index_compression: 1 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_align: 0 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.write_buffer_size: 33554432 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_write_buffer_number: 2 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression: NoCompression 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression: Disabled 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.prefix_extractor: nullptr 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.num_levels: 7 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-25T05:53:39.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression_opts.level: 32767 2024-05-25T05:53:39.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression_opts.strategy: 0 2024-05-25T05:53:39.586 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-25T05:53:39.586 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-25T05:53:39.586 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-25T05:53:39.586 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compression_opts.enabled: false 2024-05-25T05:53:39.586 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-25T05:53:39.587 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-25T05:53:39.587 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-25T05:53:39.587 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.target_file_size_base: 67108864 2024-05-25T05:53:39.587 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-25T05:53:39.587 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-25T05:53:39.587 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-25T05:53:39.587 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.arena_block_size: 4194304 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-25T05:53:39.588 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.disable_auto_compactions: 0 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-25T05:53:39.589 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.table_properties_collectors: 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.inplace_update_support: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.bloom_locality: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.max_successive_merges: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.paranoid_file_checks: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.force_consistency_checks: 1 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.report_bg_io_stats: 0 2024-05-25T05:53:39.590 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.ttl: 2592000 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.enable_blob_files: false 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.min_blob_size: 0 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.blob_file_size: 268435456 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-05-25T05:53:39.591 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716616419291275, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716616419291970, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79205, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77468, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71777, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1716616419, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "4a43ed62-fd34-4c5b-89c2-98de36120ae1", "db_session_id": "L88TTV4ND4DIJGI43KMC"}} 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716616419292319, "job": 1, "event": "recovery_finished"} 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f41c996a80 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: DB pointer 0x55f41c9a6000 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-25T05:53:39.592 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** DB Stats ** 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:53:39.593 INFO:journalctl@ceph.mon.a.smithi175.stdout: L0 2/0 79.11 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.7 0.00 0.00 1 0.000 0 0 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: Sum 2/0 79.11 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.7 0.00 0.00 1 0.000 0 0 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.7 0.00 0.00 1 0.000 0 0 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.7 0.00 0.00 1 0.000 0 0 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-25T05:53:39.594 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative compaction: 0.00 GB write, 25.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval compaction: 0.00 GB write, 25.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:53:39.595 INFO:journalctl@ceph.mon.a.smithi175.stdout: L0 2/0 79.11 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.7 0.00 0.00 1 0.000 0 0 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: Sum 2/0 79.11 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.7 0.00 0.00 1 0.000 0 0 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.7 0.00 0.00 1 0.000 0 0 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-25T05:53:39.596 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative compaction: 0.00 GB write, 25.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: starting mon.a rank 0 at public addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] at bind addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???) e1 preinit fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???).mds e1 new map 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???).mds e1 print_map 2024-05-25T05:53:39.597 INFO:journalctl@ceph.mon.a.smithi175.stdout: e1 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout: legacy client fscid: -1 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout: No filesystems configured 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-25T05:53:39.598 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expand map: {default=false} 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta from 'false' to 'false' 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expanded map: {default=false} 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expand map: {default=info} 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta from 'info' to 'info' 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expanded map: {default=info} 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expand map: {default=daemon} 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta from 'daemon' to 'daemon' 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expanded map: {default=daemon} 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expand map: {default=debug} 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta from 'debug' to 'debug' 2024-05-25T05:53:39.599 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: expand_channel_meta expanded map: {default=debug} 2024-05-25T05:53:39.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 bash[24758]: b34a4a648ba02b1e6c165033a6f89f34f239deb13bf69d4277c4adfc977ad085 2024-05-25T05:53:39.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 systemd[1]: Started Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:53:39.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-25T05:53:39.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: monmap e1: 1 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0]} 2024-05-25T05:53:39.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: fsmap 2024-05-25T05:53:39.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: osdmap e1: 0 total, 0 up, 0 in 2024-05-25T05:53:39.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:39 smithi175 ceph-mon[24776]: mgrmap e1: no daemons active 2024-05-25T05:53:40.572 INFO:teuthology.orchestra.run.smithi175.stderr:Wrote config to /etc/ceph/ceph.conf 2024-05-25T05:53:40.572 INFO:teuthology.orchestra.run.smithi175.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-25T05:53:40.572 INFO:teuthology.orchestra.run.smithi175.stderr:Creating mgr... 2024-05-25T05:53:40.572 INFO:teuthology.orchestra.run.smithi175.stderr:Verifying port 9283 ... 2024-05-25T05:53:40.752 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Failed to reset failed state of unit ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service: Unit ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service not loaded. 2024-05-25T05:53:40.760 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: Created symlink /etc/systemd/system/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968.target.wants/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@.service. 2024-05-25T05:53:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:40 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2651892194' entity='client.admin' 2024-05-25T05:53:41.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:40 smithi175 systemd[1]: Starting Ceph mgr.y for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T05:53:41.422 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 podman[24989]: 2024-05-25 05:53:41.128434424 +0000 UTC m=+0.112797691 container create 8822d7e4e7de28de687190863ea9249b20fff866cda3057f4daf4066d96a331f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, release=754, RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, architecture=x86_64, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-25T05:53:41.422 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 podman[24989]: 2024-05-25 05:53:41.032923792 +0000 UTC m=+0.017287057 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-25T05:53:41.422 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 podman[24989]: 2024-05-25 05:53:41.338977984 +0000 UTC m=+0.323341223 container init 8822d7e4e7de28de687190863ea9249b20fff866cda3057f4daf4066d96a331f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=) 2024-05-25T05:53:41.422 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 podman[24989]: 2024-05-25 05:53:41.341685563 +0000 UTC m=+0.326048817 container start 8822d7e4e7de28de687190863ea9249b20fff866cda3057f4daf4066d96a331f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-25T05:53:41.422 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 bash[24989]: 8822d7e4e7de28de687190863ea9249b20fff866cda3057f4daf4066d96a331f 2024-05-25T05:53:41.434 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: disabled 2024-05-25T05:53:41.442 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: inactive 2024-05-25T05:53:41.442 INFO:teuthology.orchestra.run.smithi175.stderr:firewalld.service is not enabled 2024-05-25T05:53:41.442 INFO:teuthology.orchestra.run.smithi175.stderr:Not possible to enable service . firewalld.service is not available 2024-05-25T05:53:41.451 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: disabled 2024-05-25T05:53:41.459 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: inactive 2024-05-25T05:53:41.459 INFO:teuthology.orchestra.run.smithi175.stderr:firewalld.service is not enabled 2024-05-25T05:53:41.459 INFO:teuthology.orchestra.run.smithi175.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-05-25T05:53:41.460 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for mgr to start... 2024-05-25T05:53:41.460 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for mgr... 2024-05-25T05:53:41.691 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 systemd[1]: Started Ceph mgr.y for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:53:41.691 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:41.487+0000 7f627b36b000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T05:53:41.691 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:41.623+0000 7f627b36b000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:53:41.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:41.723+0000 7f627b36b000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: { 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "fsid": "0bfa7ec4-1a5b-11ef-bc9b-c7b262605968", 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "health": { 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "checks": {}, 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "mutes": [] 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "quorum": [ 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 0 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: ], 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "a" 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: ], 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "monmap": { 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:41.985 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "osdmap": { 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "pgmap": { 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-25T05:53:41.986 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "fsmap": { 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "available": false, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "modules": [ 2024-05-25T05:53:41.987 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "iostat", 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "nfs", 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "restful" 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: ], 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "services": {} 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "servicemap": { 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "modified": "2024-05-25T05:53:35.189075+0000", 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "services": {} 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-25T05:53:41.988 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: } 2024-05-25T05:53:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:42 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1098477055' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-25T05:53:42.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:42.047+0000 7f627b36b000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T05:53:42.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:42.130+0000 7f627b36b000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T05:53:43.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:43.272+0000 7f627b36b000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T05:53:43.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:43.438+0000 7f627b36b000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T05:53:43.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:43.520+0000 7f627b36b000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T05:53:44.261 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:43.834+0000 7f627b36b000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T05:53:44.261 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:43.925+0000 7f627b36b000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T05:53:44.261 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:44.019+0000 7f627b36b000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T05:53:44.261 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:44.260+0000 7f627b36b000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:53:45.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:44.699+0000 7f627b36b000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T05:53:45.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:45.192+0000 7f627b36b000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T05:53:45.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:45.285+0000 7f627b36b000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T05:53:46.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:45.576+0000 7f627b36b000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T05:53:46.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:45.665+0000 7f627b36b000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T05:53:46.469 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:46.092+0000 7f627b36b000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T05:53:46.469 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:46.223+0000 7f627b36b000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T05:53:46.469 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:46.314+0000 7f627b36b000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T05:53:46.469 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:46.429+0000 7f627b36b000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T05:53:46.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:46.518+0000 7f627b36b000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T05:53:47.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:47.062+0000 7f627b36b000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T05:53:47.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:47.444+0000 7f627b36b000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T05:53:47.633 INFO:teuthology.orchestra.run.smithi175.stderr:mgr not available, waiting (1/15)... 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: Activating manager daemon y 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: mgrmap e2: y(active, starting, since 0.00318276s) 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: Manager daemon y is now available 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:53:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' 2024-05-25T05:53:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' 2024-05-25T05:53:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:47 smithi175 ceph-mon[24776]: from='mgr.14100 172.21.15.175:0/3602953947' entity='mgr.y' 2024-05-25T05:53:49.764 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:49 smithi175 ceph-mon[24776]: mgrmap e3: y(active, since 1.00883s) 2024-05-25T05:53:50.333 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:50.333 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: { 2024-05-25T05:53:50.333 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "fsid": "0bfa7ec4-1a5b-11ef-bc9b-c7b262605968", 2024-05-25T05:53:50.333 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "health": { 2024-05-25T05:53:50.333 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-25T05:53:50.333 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "checks": {}, 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "mutes": [] 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "quorum": [ 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 0 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: ], 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "a" 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: ], 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "monmap": { 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-25T05:53:50.334 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-25T05:53:50.335 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:50.335 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "osdmap": { 2024-05-25T05:53:50.335 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:50.335 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-25T05:53:50.335 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-25T05:53:50.335 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-25T05:53:50.335 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-25T05:53:50.338 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "pgmap": { 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "fsmap": { 2024-05-25T05:53:50.339 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "available": true, 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "modules": [ 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "iostat", 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "nfs", 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "restful" 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: ], 2024-05-25T05:53:50.340 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "services": {} 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "servicemap": { 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "modified": "2024-05-25T05:53:35.189075+0000", 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "services": {} 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: }, 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-25T05:53:50.341 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: } 2024-05-25T05:53:50.919 INFO:teuthology.orchestra.run.smithi175.stderr:mgr is available 2024-05-25T05:53:51.486 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:51 smithi175 ceph-mon[24776]: mgrmap e4: y(active, since 2s) 2024-05-25T05:53:51.486 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:51 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1017933110' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: [global] 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: fsid = 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: [mgr] 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-25T05:53:51.488 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 2024-05-25T05:53:51.489 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: [osd] 2024-05-25T05:53:51.489 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-25T05:53:51.489 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-25T05:53:52.109 INFO:teuthology.orchestra.run.smithi175.stderr:Enabling cephadm module... 2024-05-25T05:53:52.384 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:52 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1168529925' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-25T05:53:52.384 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:52 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1168529925' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-25T05:53:53.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:53 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/974360435' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-25T05:53:53.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setuser ceph since I am not root 2024-05-25T05:53:53.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setgroup ceph since I am not root 2024-05-25T05:53:53.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:53.473+0000 7f8e99bfb000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T05:53:53.895 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:53.603+0000 7f8e99bfb000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:53:53.895 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:53.700+0000 7f8e99bfb000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T05:53:54.146 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:54.011+0000 7f8e99bfb000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T05:53:54.146 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:54.096+0000 7f8e99bfb000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T05:53:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:54 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/974360435' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-25T05:53:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:54 smithi175 ceph-mon[24776]: mgrmap e5: y(active, since 5s) 2024-05-25T05:53:55.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:53:55 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/4092808397' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-25T05:53:55.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:55.231+0000 7f8e99bfb000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T05:53:55.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:55.395+0000 7f8e99bfb000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T05:53:55.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:55.477+0000 7f8e99bfb000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T05:53:55.580 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: { 2024-05-25T05:53:55.580 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 5, 2024-05-25T05:53:55.580 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "available": true, 2024-05-25T05:53:55.580 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-25T05:53:55.580 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-25T05:53:55.580 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: } 2024-05-25T05:53:55.883 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:55.792+0000 7f8e99bfb000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T05:53:55.883 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:55.882+0000 7f8e99bfb000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T05:53:56.216 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:55.975+0000 7f8e99bfb000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T05:53:56.216 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:56.215+0000 7f8e99bfb000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:53:57.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:56.653+0000 7f8e99bfb000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T05:53:57.525 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:57.140+0000 7f8e99bfb000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T05:53:57.525 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:57.232+0000 7f8e99bfb000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T05:53:57.525 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:57.524+0000 7f8e99bfb000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T05:53:57.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:57.614+0000 7f8e99bfb000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T05:53:58.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:58.038+0000 7f8e99bfb000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T05:53:58.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:58.170+0000 7f8e99bfb000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T05:53:58.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:58.262+0000 7f8e99bfb000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T05:53:58.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:58.383+0000 7f8e99bfb000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T05:53:58.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:58.472+0000 7f8e99bfb000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T05:53:58.949 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for the mgr to restart... 2024-05-25T05:53:58.949 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for mgr epoch 5... 2024-05-25T05:53:59.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:59.023+0000 7f8e99bfb000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T05:53:59.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:53:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:53:59.399+0000 7f8e99bfb000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T05:54:00.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:00 smithi175 ceph-mon[24776]: Active manager daemon y restarted 2024-05-25T05:54:00.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:00 smithi175 ceph-mon[24776]: Activating manager daemon y 2024-05-25T05:54:00.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:00 smithi175 ceph-mon[24776]: osdmap e2: 0 total, 0 up, 0 in 2024-05-25T05:54:01.317 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: { 2024-05-25T05:54:01.317 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-05-25T05:54:01.317 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "initialized": true 2024-05-25T05:54:01.317 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: } 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: mgrmap e6: y(active, starting, since 0.903174s) 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: Manager daemon y is now available 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:01.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:01.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:01.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:54:01.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:01 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:54:01.898 INFO:teuthology.orchestra.run.smithi175.stderr:mgr epoch 5 is available 2024-05-25T05:54:01.898 INFO:teuthology.orchestra.run.smithi175.stderr:Setting orchestrator backend to cephadm... 2024-05-25T05:54:01.988 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:54:01] ENGINE Bus STARTING 2024-05-25T05:54:01.988 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:54:01] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:54:02.256 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:54:01] ENGINE Bus STARTED 2024-05-25T05:54:02.559 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:02 smithi175 ceph-mon[24776]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-25T05:54:02.559 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:02 smithi175 ceph-mon[24776]: mgrmap e7: y(active, since 1.908s) 2024-05-25T05:54:02.559 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:02 smithi175 ceph-mon[24776]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-25T05:54:02.559 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:02 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:02.559 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:02 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:03.396 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:03 smithi175 ceph-mon[24776]: [25/May/2024:05:54:01] ENGINE Bus STARTING 2024-05-25T05:54:03.396 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:03 smithi175 ceph-mon[24776]: [25/May/2024:05:54:01] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:54:03.396 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:03 smithi175 ceph-mon[24776]: [25/May/2024:05:54:01] ENGINE Bus STARTED 2024-05-25T05:54:03.396 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:03 smithi175 ceph-mon[24776]: mgrmap e8: y(active, since 2s) 2024-05-25T05:54:03.396 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:03 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:03.396 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:03 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:03.772 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: value unchanged 2024-05-25T05:54:04.253 INFO:teuthology.orchestra.run.smithi175.stderr:Generating ssh key... 2024-05-25T05:54:04.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:04 smithi175 ceph-mon[24776]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:05.399 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:05 smithi175 ceph-mon[24776]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:05.399 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:05 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:05.399 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:05 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: Generating public/private rsa key pair. 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: Your identification has been saved in /tmp/tmprs8q1v8y/key. 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: Your public key has been saved in /tmp/tmprs8q1v8y/key.pub. 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: The key fingerprint is: 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: SHA256:3zLJgthGd2lEPzS4hixIHhCVQCpwop3jPoKBVI+2lsc ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: The key's randomart image is: 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: +---[RSA 3072]----+ 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: |o.**.. ..o | 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: |o=..* ..o . | 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: |+.+= + . ...o | 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: |+...* . o.o. . | 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: |o .+ E..S.+ | 2024-05-25T05:54:05.400 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: |.o. .+ o = o | 2024-05-25T05:54:05.401 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: |o o . + . * . | 2024-05-25T05:54:05.401 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: | . . . . o | 2024-05-25T05:54:05.401 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: | | 2024-05-25T05:54:05.401 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: +----[SHA256]-----+ 2024-05-25T05:54:06.231 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDF7+cG6TtJHljjyN7fAtpQoYuac1aIK4oO8mNhjWzX/Wh2iQMYCOGaA0pbkUd6kvVuc6WcjOxwvr3AKCD0rLnkPb0zsq0VbjFqtcHo0ux5MBS4og9FngNAhoHcam1hX8FnpzFITckgw2+jzR21UzI3tEMB5VUNTgjD0L7gXhh8YBUAq6Zhxj5R9fl6Z3yI5BzxsOvuxnr1K8n9LE3GoEuSogbcWZR5ScBFnlR9oY8zn39eTsUik4pWQCpo4ACPt+556bz7m+jNnveB+vdc+Bt6MwCgZvOiwBdyvHbvhZ+Acgq6M0Beu9I0qP6JlrQFicgHs+o3IwkyjSFeO4/KZ0OjIJ6i+VSdTXZ5L1mSQX4s3ZdQs6uBOUOXhnRbG1spccqsiPDIIBrRovgRSXooUWlUHrUU1phj8kWGA8UUi3j4XyV8EdO8FTyl/QsQbzdJ6nolXO88KnIBx6UtG2lQQv9QkzJWnWDMjDuChBZw6mNGdliwhw7VxZZh01tWe5b0530= ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:54:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:06 smithi175 ceph-mon[24776]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:06 smithi175 ceph-mon[24776]: Generating ssh key... 2024-05-25T05:54:06.793 INFO:teuthology.orchestra.run.smithi175.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-25T05:54:06.793 INFO:teuthology.orchestra.run.smithi175.stderr:Adding key to root@localhost authorized_keys... 2024-05-25T05:54:06.793 INFO:teuthology.orchestra.run.smithi175.stderr:Adding host smithi175... 2024-05-25T05:54:07.366 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:07 smithi175 ceph-mon[24776]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:08.227 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: Added host 'smithi175' with addr '172.21.15.175' 2024-05-25T05:54:08.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:08 smithi175 ceph-mon[24776]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi175", "addr": "172.21.15.175", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:08.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:08 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:08.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:08 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:08.829 INFO:teuthology.orchestra.run.smithi175.stderr:Deploying unmanaged mon service... 2024-05-25T05:54:09.498 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: Scheduled mon update... 2024-05-25T05:54:09.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:09 smithi175 ceph-mon[24776]: Deploying cephadm binary to smithi175 2024-05-25T05:54:09.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:09 smithi175 ceph-mon[24776]: Added host smithi175 2024-05-25T05:54:10.056 INFO:teuthology.orchestra.run.smithi175.stderr:Deploying unmanaged mgr service... 2024-05-25T05:54:10.532 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:10 smithi175 ceph-mon[24776]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:10.532 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:10 smithi175 ceph-mon[24776]: Saving service mon spec with placement count:5 2024-05-25T05:54:10.532 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:10 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:10.631 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-05-25T05:54:11.950 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:11 smithi175 ceph-mon[24776]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:11.950 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:11 smithi175 ceph-mon[24776]: Saving service mgr spec with placement count:2 2024-05-25T05:54:11.950 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:11 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:11.950 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:11 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:12 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:12 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/771483149' entity='client.admin' 2024-05-25T05:54:14.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:14 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:14.966 INFO:teuthology.orchestra.run.smithi175.stderr:Enabling the dashboard module... 2024-05-25T05:54:15.533 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:15 smithi175 ceph-mon[24776]: from='mgr.14116 172.21.15.175:0/4006166307' entity='mgr.y' 2024-05-25T05:54:15.533 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:15 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2127531963' entity='client.admin' 2024-05-25T05:54:16.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:16 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/631709324' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-25T05:54:16.397 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setuser ceph since I am not root 2024-05-25T05:54:16.397 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setgroup ceph since I am not root 2024-05-25T05:54:16.741 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:16.511+0000 7f2fb148a000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T05:54:16.742 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:16.641+0000 7f2fb148a000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:54:16.742 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:16.741+0000 7f2fb148a000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T05:54:17.310 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:17.048+0000 7f2fb148a000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T05:54:17.310 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:17.141+0000 7f2fb148a000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T05:54:17.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:17 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/631709324' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-25T05:54:17.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:17 smithi175 ceph-mon[24776]: mgrmap e9: y(active, since 16s) 2024-05-25T05:54:17.814 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: { 2024-05-25T05:54:17.814 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "epoch": 9, 2024-05-25T05:54:17.814 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "available": true, 2024-05-25T05:54:17.814 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-25T05:54:17.814 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-25T05:54:17.814 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: } 2024-05-25T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:18 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2734127062' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-25T05:54:18.408 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:18.288+0000 7f2fb148a000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T05:54:18.410 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for the mgr to restart... 2024-05-25T05:54:18.410 INFO:teuthology.orchestra.run.smithi175.stderr:Waiting for mgr epoch 9... 2024-05-25T05:54:18.681 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:18.460+0000 7f2fb148a000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T05:54:18.682 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:18.542+0000 7f2fb148a000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T05:54:18.971 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:18.873+0000 7f2fb148a000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T05:54:18.971 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:18.970+0000 7f2fb148a000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T05:54:19.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:19.070+0000 7f2fb148a000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T05:54:19.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:19.327+0000 7f2fb148a000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:54:19.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:19.781+0000 7f2fb148a000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T05:54:20.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:20.275+0000 7f2fb148a000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T05:54:20.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:20.370+0000 7f2fb148a000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T05:54:21.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:20.663+0000 7f2fb148a000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T05:54:21.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:20.753+0000 7f2fb148a000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T05:54:21.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:21.176+0000 7f2fb148a000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T05:54:21.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:21.308+0000 7f2fb148a000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T05:54:21.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:21.402+0000 7f2fb148a000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T05:54:21.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:21.515+0000 7f2fb148a000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T05:54:21.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:21.603+0000 7f2fb148a000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T05:54:22.545 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:22.155+0000 7f2fb148a000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T05:54:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:22 smithi175 ceph-mon[24776]: Active manager daemon y restarted 2024-05-25T05:54:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:22 smithi175 ceph-mon[24776]: Activating manager daemon y 2024-05-25T05:54:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:22 smithi175 ceph-mon[24776]: osdmap e3: 0 total, 0 up, 0 in 2024-05-25T05:54:22.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:54:22.544+0000 7f2fb148a000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T05:54:23.614 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: { 2024-05-25T05:54:23.614 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-05-25T05:54:23.614 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: "initialized": true 2024-05-25T05:54:23.614 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: } 2024-05-25T05:54:23.867 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: mgrmap e10: y(active, starting, since 0.0553026s) 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: Manager daemon y is now available 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:54:23.868 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:54:24.225 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:54:23] ENGINE Bus STARTING 2024-05-25T05:54:24.225 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:54:23] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:54:24.225 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:54:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:54:23] ENGINE Bus STARTED 2024-05-25T05:54:24.227 INFO:teuthology.orchestra.run.smithi175.stderr:mgr epoch 9 is available 2024-05-25T05:54:24.227 INFO:teuthology.orchestra.run.smithi175.stderr:Generating a dashboard self-signed certificate... 2024-05-25T05:54:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:24 smithi175 ceph-mon[24776]: mgrmap e11: y(active, since 1.06056s) 2024-05-25T05:54:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:24 smithi175 ceph-mon[24776]: [25/May/2024:05:54:23] ENGINE Bus STARTING 2024-05-25T05:54:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:24 smithi175 ceph-mon[24776]: [25/May/2024:05:54:23] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:54:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:24 smithi175 ceph-mon[24776]: [25/May/2024:05:54:23] ENGINE Bus STARTED 2024-05-25T05:54:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:24 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:24 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:25.100 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: Self-signed certificate created 2024-05-25T05:54:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:25 smithi175 ceph-mon[24776]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:25 smithi175 ceph-mon[24776]: mgrmap e12: y(active, since 2s) 2024-05-25T05:54:26.750 INFO:teuthology.orchestra.run.smithi175.stderr:Creating initial admin user... 2024-05-25T05:54:27.626 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$30oChuC7Yibf4Lgfgbvp2OtiRXRF7ftfIG9.7okmK0Q1fuhr3v22C", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1716616467, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-25T05:54:28.224 INFO:teuthology.orchestra.run.smithi175.stderr:Fetching dashboard port number... 2024-05-25T05:54:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:28 smithi175 ceph-mon[24776]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:28.843 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: 8443 2024-05-25T05:54:29.531 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: disabled 2024-05-25T05:54:29.538 INFO:teuthology.orchestra.run.smithi175.stderr:systemctl: inactive 2024-05-25T05:54:29.539 INFO:teuthology.orchestra.run.smithi175.stderr:firewalld.service is not enabled 2024-05-25T05:54:29.539 INFO:teuthology.orchestra.run.smithi175.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-25T05:54:29.541 INFO:teuthology.orchestra.run.smithi175.stderr:Ceph Dashboard is now available at: 2024-05-25T05:54:29.541 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:29.541 INFO:teuthology.orchestra.run.smithi175.stderr: URL: https://smithi175.front.sepia.ceph.com:8443/ 2024-05-25T05:54:29.541 INFO:teuthology.orchestra.run.smithi175.stderr: User: admin 2024-05-25T05:54:29.541 INFO:teuthology.orchestra.run.smithi175.stderr: Password: x2ogdgaixf 2024-05-25T05:54:29.541 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:29.542 INFO:teuthology.orchestra.run.smithi175.stderr:Enabling autotune for osd_memory_target 2024-05-25T05:54:29.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:29 smithi175 ceph-mon[24776]: mgrmap e13: y(active, since 6s) 2024-05-25T05:54:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:29 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3477024336' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-25T05:54:31.492 INFO:teuthology.orchestra.run.smithi175.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-05-25T05:54:32.111 INFO:teuthology.orchestra.run.smithi175.stderr:You can access the Ceph CLI with: 2024-05-25T05:54:32.111 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:32.111 INFO:teuthology.orchestra.run.smithi175.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-25T05:54:32.111 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:32.111 INFO:teuthology.orchestra.run.smithi175.stderr:Please consider enabling telemetry to help improve Ceph: 2024-05-25T05:54:32.111 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:32.111 INFO:teuthology.orchestra.run.smithi175.stderr: ceph telemetry on 2024-05-25T05:54:32.112 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:32.112 INFO:teuthology.orchestra.run.smithi175.stderr:For more information see: 2024-05-25T05:54:32.112 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:32.112 INFO:teuthology.orchestra.run.smithi175.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-05-25T05:54:32.112 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T05:54:32.112 INFO:teuthology.orchestra.run.smithi175.stderr:Bootstrap complete. 2024-05-25T05:54:32.145 INFO:tasks.cephadm:Fetching config... 2024-05-25T05:54:32.146 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:54:32.146 DEBUG:teuthology.orchestra.run.smithi175:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-25T05:54:32.161 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-25T05:54:32.161 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:54:32.162 DEBUG:teuthology.orchestra.run.smithi175:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-25T05:54:32.217 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-25T05:54:32.218 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:54:32.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/keyring of=/dev/stdout 2024-05-25T05:54:32.282 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-25T05:54:32.282 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:54:32.282 DEBUG:teuthology.orchestra.run.smithi175:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-25T05:54:32.338 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-25T05:54:32.338 DEBUG:teuthology.orchestra.run.smithi175:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDF7+cG6TtJHljjyN7fAtpQoYuac1aIK4oO8mNhjWzX/Wh2iQMYCOGaA0pbkUd6kvVuc6WcjOxwvr3AKCD0rLnkPb0zsq0VbjFqtcHo0ux5MBS4og9FngNAhoHcam1hX8FnpzFITckgw2+jzR21UzI3tEMB5VUNTgjD0L7gXhh8YBUAq6Zhxj5R9fl6Z3yI5BzxsOvuxnr1K8n9LE3GoEuSogbcWZR5ScBFnlR9oY8zn39eTsUik4pWQCpo4ACPt+556bz7m+jNnveB+vdc+Bt6MwCgZvOiwBdyvHbvhZ+Acgq6M0Beu9I0qP6JlrQFicgHs+o3IwkyjSFeO4/KZ0OjIJ6i+VSdTXZ5L1mSQX4s3ZdQs6uBOUOXhnRbG1spccqsiPDIIBrRovgRSXooUWlUHrUU1phj8kWGA8UUi3j4XyV8EdO8FTyl/QsQbzdJ6nolXO88KnIBx6UtG2lQQv9QkzJWnWDMjDuChBZw6mNGdliwhw7VxZZh01tWe5b0530= ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-25T05:54:32.416 INFO:teuthology.orchestra.run.smithi175.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDF7+cG6TtJHljjyN7fAtpQoYuac1aIK4oO8mNhjWzX/Wh2iQMYCOGaA0pbkUd6kvVuc6WcjOxwvr3AKCD0rLnkPb0zsq0VbjFqtcHo0ux5MBS4og9FngNAhoHcam1hX8FnpzFITckgw2+jzR21UzI3tEMB5VUNTgjD0L7gXhh8YBUAq6Zhxj5R9fl6Z3yI5BzxsOvuxnr1K8n9LE3GoEuSogbcWZR5ScBFnlR9oY8zn39eTsUik4pWQCpo4ACPt+556bz7m+jNnveB+vdc+Bt6MwCgZvOiwBdyvHbvhZ+Acgq6M0Beu9I0qP6JlrQFicgHs+o3IwkyjSFeO4/KZ0OjIJ6i+VSdTXZ5L1mSQX4s3ZdQs6uBOUOXhnRbG1spccqsiPDIIBrRovgRSXooUWlUHrUU1phj8kWGA8UUi3j4XyV8EdO8FTyl/QsQbzdJ6nolXO88KnIBx6UtG2lQQv9QkzJWnWDMjDuChBZw6mNGdliwhw7VxZZh01tWe5b0530= ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:54:32.428 DEBUG:teuthology.orchestra.run.smithi179:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDF7+cG6TtJHljjyN7fAtpQoYuac1aIK4oO8mNhjWzX/Wh2iQMYCOGaA0pbkUd6kvVuc6WcjOxwvr3AKCD0rLnkPb0zsq0VbjFqtcHo0ux5MBS4og9FngNAhoHcam1hX8FnpzFITckgw2+jzR21UzI3tEMB5VUNTgjD0L7gXhh8YBUAq6Zhxj5R9fl6Z3yI5BzxsOvuxnr1K8n9LE3GoEuSogbcWZR5ScBFnlR9oY8zn39eTsUik4pWQCpo4ACPt+556bz7m+jNnveB+vdc+Bt6MwCgZvOiwBdyvHbvhZ+Acgq6M0Beu9I0qP6JlrQFicgHs+o3IwkyjSFeO4/KZ0OjIJ6i+VSdTXZ5L1mSQX4s3ZdQs6uBOUOXhnRbG1spccqsiPDIIBrRovgRSXooUWlUHrUU1phj8kWGA8UUi3j4XyV8EdO8FTyl/QsQbzdJ6nolXO88KnIBx6UtG2lQQv9QkzJWnWDMjDuChBZw6mNGdliwhw7VxZZh01tWe5b0530= ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-25T05:54:32.469 INFO:teuthology.orchestra.run.smithi179.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDF7+cG6TtJHljjyN7fAtpQoYuac1aIK4oO8mNhjWzX/Wh2iQMYCOGaA0pbkUd6kvVuc6WcjOxwvr3AKCD0rLnkPb0zsq0VbjFqtcHo0ux5MBS4og9FngNAhoHcam1hX8FnpzFITckgw2+jzR21UzI3tEMB5VUNTgjD0L7gXhh8YBUAq6Zhxj5R9fl6Z3yI5BzxsOvuxnr1K8n9LE3GoEuSogbcWZR5ScBFnlR9oY8zn39eTsUik4pWQCpo4ACPt+556bz7m+jNnveB+vdc+Bt6MwCgZvOiwBdyvHbvhZ+Acgq6M0Beu9I0qP6JlrQFicgHs+o3IwkyjSFeO4/KZ0OjIJ6i+VSdTXZ5L1mSQX4s3ZdQs6uBOUOXhnRbG1spccqsiPDIIBrRovgRSXooUWlUHrUU1phj8kWGA8UUi3j4XyV8EdO8FTyl/QsQbzdJ6nolXO88KnIBx6UtG2lQQv9QkzJWnWDMjDuChBZw6mNGdliwhw7VxZZh01tWe5b0530= ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:54:32.482 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-25T05:54:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:32 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/465241379' entity='client.admin' 2024-05-25T05:54:35.153 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:35 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:35.153 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:35 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi175", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:54:35.153 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:35 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:35.153 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:35 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:35.153 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:35 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:36.673 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-25T05:54:36.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-25T05:54:37.262 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:37 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1264772459' entity='client.admin' 2024-05-25T05:54:37.263 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:37.263 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:39.263 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi179 2024-05-25T05:54:39.264 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-05-25T05:54:39.264 DEBUG:teuthology.orchestra.run.smithi179:> dd of=/etc/ceph/ceph.conf 2024-05-25T05:54:39.284 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-05-25T05:54:39.285 DEBUG:teuthology.orchestra.run.smithi179:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:54:39.343 INFO:tasks.cephadm:Adding host smithi179 to orchestrator... 2024-05-25T05:54:39.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch host add smithi179 2024-05-25T05:54:39.752 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:39.752 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:39.752 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:39.752 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:54:39.752 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:54:39.752 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T05:54:39.752 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:54:39.753 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:39.753 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:41.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:41 smithi175 ceph-mon[24776]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi179", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:42.059 INFO:teuthology.orchestra.run.smithi175.stdout:Added host 'smithi179' with addr '172.21.15.179' 2024-05-25T05:54:42.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch host ls --format=json 2024-05-25T05:54:43.188 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: Deploying cephadm binary to smithi179 2024-05-25T05:54:43.189 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:43.189 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: Added host smithi179 2024-05-25T05:54:43.189 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:43.189 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:54:43.189 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:54:43.189 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:43.189 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:43 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:44.312 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:54:44.312 INFO:teuthology.orchestra.run.smithi175.stdout:[{"addr": "172.21.15.175", "hostname": "smithi175", "labels": [], "status": ""}, {"addr": "172.21.15.179", "hostname": "smithi179", "labels": [], "status": ""}] 2024-05-25T05:54:44.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:44 smithi175 ceph-mon[24776]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:44.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:44.921 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-25T05:54:44.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd crush tunables default 2024-05-25T05:54:45.484 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:45 smithi175 ceph-mon[24776]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-25T05:54:46.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:46 smithi175 ceph-mon[24776]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:47.468 INFO:teuthology.orchestra.run.smithi175.stderr:adjusted tunables profile to default 2024-05-25T05:54:47.733 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:47 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:47.733 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:47 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:47.734 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:47 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi179", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:54:47.734 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:47 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:47.734 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:47 smithi175 ceph-mon[24776]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:47.734 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:47 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3566069434' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-25T05:54:48.029 INFO:tasks.cephadm:Adding mon.a on smithi175 2024-05-25T05:54:48.041 INFO:tasks.cephadm:Adding mon.c on smithi175 2024-05-25T05:54:48.042 INFO:tasks.cephadm:Adding mon.b on smithi179 2024-05-25T05:54:48.042 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch apply mon '3;smithi175:172.21.15.175=a;smithi175:[v2:172.21.15.175:3301,v1:172.21.15.175:6790]=c;smithi179:172.21.15.179=b' 2024-05-25T05:54:48.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:48 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3566069434' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-25T05:54:48.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:48 smithi175 ceph-mon[24776]: osdmap e4: 0 total, 0 up, 0 in 2024-05-25T05:54:49.880 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled mon update... 2024-05-25T05:54:50.484 DEBUG:teuthology.orchestra.run.smithi175:mon.c> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.c.service 2024-05-25T05:54:50.487 DEBUG:teuthology.orchestra.run.smithi179:mon.b> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.b.service 2024-05-25T05:54:50.490 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-25T05:54:50.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:54:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:54:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:51 smithi175 ceph-mon[24776]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi175:172.21.15.175=a;smithi175:[v2:172.21.15.175:3301,v1:172.21.15.175:6790]=c;smithi179:172.21.15.179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:54:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:51 smithi175 ceph-mon[24776]: Saving service mon spec with placement smithi175:172.21.15.175=a;smithi175:[v2:172.21.15.175:3301,v1:172.21.15.175:6790]=c;smithi179:172.21.15.179=b;count:3 2024-05-25T05:54:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:51 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:54:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:51 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:54:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:51 smithi175 ceph-mon[24776]: Deploying daemon mon.c on smithi175 2024-05-25T05:54:52.348 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:54:52.348 INFO:teuthology.orchestra.run.smithi179.stdout:{"epoch":1,"fsid":"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968","modified":"2024-05-25T05:53:32.512282Z","created":"2024-05-25T05:53:32.512282Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3300","nonce":0},{"type":"v1","addr":"172.21.15.175:6789","nonce":0}]},"addr":"172.21.15.175:6789/0","public_addr":"172.21.15.175:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-25T05:54:52.350 INFO:teuthology.orchestra.run.smithi179.stderr:dumped monmap epoch 1 2024-05-25T05:54:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:52 smithi175 ceph-mon[24776]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:53.549 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:53 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/1184757192' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:54:53.550 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:53 smithi175 systemd[1]: Started Ceph mon.c for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:54:53.846 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-25T05:54:53.847 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-05-25T05:54:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: mon.a calling monitor election 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: mon.c calling monitor election 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: monmap e2: 2 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: fsmap 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: osdmap e4: 0 total, 0 up, 0 in 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: mgrmap e13: y(active, since 35s) 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: overall HEALTH_OK 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:54:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: mon.a calling monitor election 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: mon.c calling monitor election 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: monmap e2: 2 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: fsmap 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: osdmap e4: 0 total, 0 up, 0 in 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: mgrmap e13: y(active, since 35s) 2024-05-25T05:54:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: overall HEALTH_OK 2024-05-25T05:54:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:54:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:54:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:54:58.833 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:54:58.834 INFO:teuthology.orchestra.run.smithi179.stdout:{"epoch":2,"fsid":"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968","modified":"2024-05-25T05:54:53.466977Z","created":"2024-05-25T05:53:32.512282Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3300","nonce":0},{"type":"v1","addr":"172.21.15.175:6789","nonce":0}]},"addr":"172.21.15.175:6789/0","public_addr":"172.21.15.175:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3301","nonce":0},{"type":"v1","addr":"172.21.15.175:6790","nonce":0}]},"addr":"172.21.15.175:6790/0","public_addr":"172.21.15.175:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-25T05:54:58.836 INFO:teuthology.orchestra.run.smithi179.stderr:dumped monmap epoch 2 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[24776]: Deploying daemon mon.b on smithi179 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[24776]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/869140520' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[29436]: Deploying daemon mon.b on smithi179 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[29436]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/869140520' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:54:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:54:59 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:00.510 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-25T05:55:00.511 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-05-25T05:55:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:01 smithi175 ceph-mon[29436]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:01 smithi175 ceph-mon[24776]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:02 smithi179 systemd[1]: Starting Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T05:55:02.953 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:55:02.954 INFO:teuthology.orchestra.run.smithi179.stdout:{"epoch":2,"fsid":"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968","modified":"2024-05-25T05:54:53.466977Z","created":"2024-05-25T05:53:32.512282Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3300","nonce":0},{"type":"v1","addr":"172.21.15.175:6789","nonce":0}]},"addr":"172.21.15.175:6789/0","public_addr":"172.21.15.175:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3301","nonce":0},{"type":"v1","addr":"172.21.15.175:6790","nonce":0}]},"addr":"172.21.15.175:6790/0","public_addr":"172.21.15.175:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-25T05:55:02.955 INFO:teuthology.orchestra.run.smithi179.stderr:dumped monmap epoch 2 2024-05-25T05:55:03.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:02 smithi179 podman[25592]: 2024-05-25 05:55:02.811087196 +0000 UTC m=+0.020502172 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-25T05:55:03.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:02 smithi179 podman[25592]: 2024-05-25 05:55:02.916393101 +0000 UTC m=+0.125808056 container create bc465d7efed04e0ab8a9849ab7d054c29f4b6fb932a87499592006dd6645c991 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, release=754, distribution-scope=public, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-type=git, io.buildah.version=1.19.8, ceph=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True) 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 podman[25592]: 2024-05-25 05:55:03.311725292 +0000 UTC m=+0.521140256 container init bc465d7efed04e0ab8a9849ab7d054c29f4b6fb932a87499592006dd6645c991 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, distribution-scope=public, ceph=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, release=754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 podman[25592]: 2024-05-25 05:55:03.315956954 +0000 UTC m=+0.525371909 container start bc465d7efed04e0ab8a9849ab7d054c29f4b6fb932a87499592006dd6645c991 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, release=754, ceph=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=centos-stream-container, vcs-type=git) 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: set uid:gid to 167:167 (ceph:ceph) 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: pidfile_write: ignore empty --pid-file 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: load: jerasure load: lrc 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: RocksDB version: 6.15.5 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Compile date Apr 18 2022 2024-05-25T05:55:03.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: DB SUMMARY 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: DB Session ID: G4EHY9YS7AFNRVM3KDV8 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: CURRENT file: CURRENT 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: IDENTITY file: IDENTITY 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.error_if_exists: 0 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.create_if_missing: 0 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.paranoid_checks: 1 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.env: 0x55c071cc6860 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.fs: Posix File System 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.info_log: 0x55c072909dc0 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_file_opening_threads: 16 2024-05-25T05:55:03.707 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.statistics: (nil) 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.use_fsync: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_log_file_size: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.keep_log_file_num: 1000 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.recycle_log_file_num: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.allow_fallocate: 1 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.allow_mmap_reads: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.allow_mmap_writes: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.use_direct_reads: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.create_missing_column_families: 0 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.db_log_dir: 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-05-25T05:55:03.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-25T05:55:03.709 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-25T05:55:03.709 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-25T05:55:03.709 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-25T05:55:03.709 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-25T05:55:03.709 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-25T05:55:03.710 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.advise_random_on_open: 1 2024-05-25T05:55:03.710 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.db_write_buffer_size: 0 2024-05-25T05:55:03.710 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.write_buffer_manager: 0x55c0729fa240 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.rate_limiter: (nil) 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.wal_recovery_mode: 2 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.enable_thread_tracking: 0 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.enable_pipelined_write: 0 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.unordered_write: 0 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-25T05:55:03.711 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.row_cache: None 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.wal_filter: None 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.allow_ingest_behind: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.preserve_deletes: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.two_write_queues: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.manual_wal_flush: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.atomic_flush: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.log_readahead_size: 0 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-25T05:55:03.712 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.best_efforts_recovery: 0 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.allow_data_in_errors: 0 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.db_host_id: __hostname__ 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_background_jobs: 2 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_background_compactions: -1 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_subcompactions: 1 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_total_wal_size: 0 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-25T05:55:03.713 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_open_files: -1 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bytes_per_sync: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_readahead_size: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_background_flushes: -1 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Compression algorithms supported: 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kZSTD supported: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kXpressCompression supported: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kLZ4HCCompression supported: 1 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kLZ4Compression supported: 1 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kBZip2Compression supported: 0 2024-05-25T05:55:03.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kZlibCompression supported: 1 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: kSnappyCompression supported: 1 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.merge_operator: 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_filter: None 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_filter_factory: None 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.sst_partitioner_factory: None 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c0728d5d00) 2024-05-25T05:55:03.715 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks: 1 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_top_level_index_and_filter: 1 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_type: 0 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_index_type: 0 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_shortening: 1 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: hash_index_allow_collision: 1 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: checksum: 1 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: no_block_cache: 0 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache: 0x55c072940170 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_name: BinnedLRUCache 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_options: 2024-05-25T05:55:03.716 INFO:journalctl@ceph.mon.b.smithi179.stdout: capacity : 536870912 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: num_shard_bits : 4 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: strict_capacity_limit : 0 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: high_pri_pool_ratio: 0.000 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_compressed: (nil) 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: persistent_cache: (nil) 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size: 4096 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size_deviation: 10 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_restart_interval: 16 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_block_restart_interval: 1 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: metadata_block_size: 4096 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: partition_filters: 0 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_delta_encoding: 1 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: whole_key_filtering: 1 2024-05-25T05:55:03.717 INFO:journalctl@ceph.mon.b.smithi179.stdout: verify_compression: 0 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout: read_amp_bytes_per_bit: 0 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout: format_version: 4 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_index_compression: 1 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_align: 0 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.write_buffer_size: 33554432 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_write_buffer_number: 2 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression: NoCompression 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression: Disabled 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.prefix_extractor: nullptr 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.num_levels: 7 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-25T05:55:03.718 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression_opts.level: 32767 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression_opts.strategy: 0 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-25T05:55:03.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compression_opts.enabled: false 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.target_file_size_base: 67108864 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-25T05:55:03.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.arena_block_size: 4194304 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.disable_auto_compactions: 0 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-25T05:55:03.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.table_properties_collectors: 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.inplace_update_support: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.bloom_locality: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.max_successive_merges: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.paranoid_file_checks: 0 2024-05-25T05:55:03.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.force_consistency_checks: 1 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.report_bg_io_stats: 0 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.ttl: 2592000 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.enable_blob_files: false 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.min_blob_size: 0 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.blob_file_size: 268435456 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716616503343051, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-25T05:55:03.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716616503343369, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1716616503, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ec3f1c41-8be2-45f9-b4c9-9bf357f96ea3", "db_session_id": "G4EHY9YS7AFNRVM3KDV8"}} 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716616503343679, "job": 1, "event": "recovery_finished"} 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55c072926a80 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: DB pointer 0x55c072936000 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** DB Stats ** 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T05:55:03.724 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-05-25T05:55:03.725 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:55:03.726 INFO:journalctl@ceph.mon.b.smithi179.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:55:03.727 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: using public_addr v2:172.21.15.179:0/0 -> [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: starting mon.b rank -1 at public addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] at bind addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(???) e0 preinit fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).mds e1 new map 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).mds e1 print_map 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: e1 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: legacy client fscid: -1 2024-05-25T05:55:03.728 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout: No filesystems configured 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi175:172.21.15.175=a;smithi175:[v2:172.21.15.175:3301,v1:172.21.15.175:6790]=c;smithi179:172.21.15.179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: Saving service mon spec with placement smithi175:172.21.15.175=a;smithi175:[v2:172.21.15.175:3301,v1:172.21.15.175:6790]=c;smithi179:172.21.15.179=b;count:3 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:55:03.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: Deploying daemon mon.c on smithi175 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/1184757192' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.a calling monitor election 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.c calling monitor election 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:03.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: monmap e2: 2 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: fsmap 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: osdmap e4: 0 total, 0 up, 0 in 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mgrmap e13: y(active, since 35s) 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: overall HEALTH_OK 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: Deploying daemon mon.b on smithi179 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/869140520' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-05-25T05:55:03.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 ceph-mon[25623]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-25T05:55:03.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 bash[25592]: bc465d7efed04e0ab8a9849ab7d054c29f4b6fb932a87499592006dd6645c991 2024-05-25T05:55:03.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:03 smithi179 systemd[1]: Started Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:55:04.733 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-25T05:55:04.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: mon.a calling monitor election 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: mon.c calling monitor election 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: mon.b calling monitor election 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: monmap e3: 3 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: fsmap 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: osdmap e4: 0 total, 0 up, 0 in 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: mgrmap e13: y(active, since 45s) 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: overall HEALTH_OK 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:08.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:55:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: mon.a calling monitor election 2024-05-25T05:55:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: mon.c calling monitor election 2024-05-25T05:55:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: mon.b calling monitor election 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: monmap e3: 3 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: fsmap 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: osdmap e4: 0 total, 0 up, 0 in 2024-05-25T05:55:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: mgrmap e13: y(active, since 45s) 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: overall HEALTH_OK 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: mon.a calling monitor election 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: mon.c calling monitor election 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:55:08.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: mon.b calling monitor election 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: monmap e3: 3 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: fsmap 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: osdmap e4: 0 total, 0 up, 0 in 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: mgrmap e13: y(active, since 45s) 2024-05-25T05:55:08.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: overall HEALTH_OK 2024-05-25T05:55:08.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:08.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:08.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:08.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:09.505 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-05-25T05:55:09.505 INFO:teuthology.orchestra.run.smithi179.stdout:{"epoch":3,"fsid":"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968","modified":"2024-05-25T05:55:03.364297Z","created":"2024-05-25T05:53:32.512282Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3300","nonce":0},{"type":"v1","addr":"172.21.15.175:6789","nonce":0}]},"addr":"172.21.15.175:6789/0","public_addr":"172.21.15.175:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3301","nonce":0},{"type":"v1","addr":"172.21.15.175:6790","nonce":0}]},"addr":"172.21.15.175:6790/0","public_addr":"172.21.15.175:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:3300","nonce":0},{"type":"v1","addr":"172.21.15.179:6789","nonce":0}]},"addr":"172.21.15.179:6789/0","public_addr":"172.21.15.179:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-05-25T05:55:09.507 INFO:teuthology.orchestra.run.smithi179.stderr:dumped monmap epoch 3 2024-05-25T05:55:10.082 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-25T05:55:10.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph config generate-minimal-conf 2024-05-25T05:55:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:55:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/1039542116' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:55:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:10.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/1039542116' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:10.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/1039542116' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:10.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:11 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T05:55:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:11 smithi179 ceph-mon[25623]: Reconfiguring mon.a (unknown last config time)... 2024-05-25T05:55:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:11 smithi179 ceph-mon[25623]: Reconfiguring daemon mon.a on smithi175 2024-05-25T05:55:11.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:11 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:11.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:11 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:11.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:11 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:11.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:11 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T05:55:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[24776]: Reconfiguring mon.a (unknown last config time)... 2024-05-25T05:55:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[24776]: Reconfiguring daemon mon.a on smithi175 2024-05-25T05:55:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T05:55:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[29436]: Reconfiguring mon.a (unknown last config time)... 2024-05-25T05:55:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[29436]: Reconfiguring daemon mon.a on smithi175 2024-05-25T05:55:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:11 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:12 smithi179 ceph-mon[25623]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:12 smithi179 ceph-mon[25623]: Reconfiguring mon.c (monmap changed)... 2024-05-25T05:55:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:12 smithi179 ceph-mon[25623]: Reconfiguring daemon mon.c on smithi175 2024-05-25T05:55:12.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:12 smithi175 ceph-mon[24776]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:12 smithi175 ceph-mon[24776]: Reconfiguring mon.c (monmap changed)... 2024-05-25T05:55:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:12 smithi175 ceph-mon[24776]: Reconfiguring daemon mon.c on smithi175 2024-05-25T05:55:12.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:12 smithi175 ceph-mon[29436]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:12.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:12 smithi175 ceph-mon[29436]: Reconfiguring mon.c (monmap changed)... 2024-05-25T05:55:12.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:12 smithi175 ceph-mon[29436]: Reconfiguring daemon mon.c on smithi175 2024-05-25T05:55:13.734 INFO:teuthology.orchestra.run.smithi175.stdout:# minimal ceph.conf for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:55:13.734 INFO:teuthology.orchestra.run.smithi175.stdout:[global] 2024-05-25T05:55:13.734 INFO:teuthology.orchestra.run.smithi175.stdout: fsid = 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T05:55:13.734 INFO:teuthology.orchestra.run.smithi175.stdout: mon_host = [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] 2024-05-25T05:55:13.896 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: Reconfiguring mon.b (monmap changed)... 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[29436]: Reconfiguring daemon mon.b on smithi179 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:13.897 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: Reconfiguring mon.b (monmap changed)... 2024-05-25T05:55:13.898 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:13.898 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:13.898 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:13.898 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:13 smithi175 ceph-mon[24776]: Reconfiguring daemon mon.b on smithi179 2024-05-25T05:55:13.968 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:13.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:13.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:13.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: Reconfiguring mon.b (monmap changed)... 2024-05-25T05:55:13.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T05:55:13.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T05:55:13.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:13.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:13 smithi179 ceph-mon[25623]: Reconfiguring daemon mon.b on smithi179 2024-05-25T05:55:14.352 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-25T05:55:14.352 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:55:14.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.conf 2024-05-25T05:55:14.379 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:55:14.379 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:55:14.444 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-05-25T05:55:14.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/ceph/ceph.conf 2024-05-25T05:55:14.472 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-05-25T05:55:14.472 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:55:14.538 INFO:tasks.cephadm:Adding mgr.y on smithi175 2024-05-25T05:55:14.538 INFO:tasks.cephadm:Adding mgr.x on smithi179 2024-05-25T05:55:14.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch apply mgr '2;smithi175=y;smithi179=x' 2024-05-25T05:55:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:14 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1944951498' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:14.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1944951498' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1944951498' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:16.284 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled mgr update... 2024-05-25T05:55:16.335 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:16.335 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T05:55:16.336 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:16 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T05:55:16.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-25T05:55:16.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T05:55:16.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:16 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:16.887 DEBUG:teuthology.orchestra.run.smithi179:mgr.x> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service 2024-05-25T05:55:16.890 INFO:tasks.cephadm:Deploying OSDs... 2024-05-25T05:55:16.890 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:55:16.890 DEBUG:teuthology.orchestra.run.smithi175:> dd if=/scratch_devs of=/dev/stdout 2024-05-25T05:55:16.909 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-25T05:55:16.909 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_1 2024-05-25T05:55:16.968 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-25T05:55:16.968 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:16.968 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-05-25T05:55:16.968 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:16.968 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:16.968 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-05-25 05:54:52.900942148 +0000 2024-05-25T05:55:16.968 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-05-25 05:52:16.085452785 +0000 2024-05-25T05:55:16.969 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-05-25 05:52:16.085452785 +0000 2024-05-25T05:55:16.969 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-05-25 05:52:16.085452785 +0000 2024-05-25T05:55:16.969 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-25T05:55:17.036 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-05-25T05:55:17.036 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-05-25T05:55:17.036 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000228375 s, 2.2 MB/s 2024-05-25T05:55:17.037 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-25T05:55:17.096 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_2 2024-05-25T05:55:17.153 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-25T05:55:17.153 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:17.153 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-05-25T05:55:17.153 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:17.153 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:17.153 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-05-25 05:54:52.900942148 +0000 2024-05-25T05:55:17.153 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-05-25 05:52:16.416447639 +0000 2024-05-25T05:55:17.154 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-05-25 05:52:16.416447639 +0000 2024-05-25T05:55:17.154 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-05-25 05:52:16.416447639 +0000 2024-05-25T05:55:17.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-25T05:55:17.218 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-05-25T05:55:17.218 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-05-25T05:55:17.218 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000220008 s, 2.3 MB/s 2024-05-25T05:55:17.219 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-25T05:55:17.276 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_3 2024-05-25T05:55:17.332 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-25T05:55:17.332 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:17.332 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-05-25T05:55:17.332 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:17.332 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:17.333 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-05-25 05:54:52.900942148 +0000 2024-05-25T05:55:17.333 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-05-25 05:52:16.744442539 +0000 2024-05-25T05:55:17.333 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-05-25 05:52:16.744442539 +0000 2024-05-25T05:55:17.333 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-05-25 05:52:16.744442539 +0000 2024-05-25T05:55:17.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-25T05:55:17.396 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-05-25T05:55:17.396 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-05-25T05:55:17.397 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000204475 s, 2.5 MB/s 2024-05-25T05:55:17.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:17 smithi179 ceph-mon[25623]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi175=y;smithi179=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:17.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:17 smithi179 ceph-mon[25623]: Saving service mgr spec with placement smithi175=y;smithi179=x;count:2 2024-05-25T05:55:17.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:17 smithi179 ceph-mon[25623]: Deploying daemon mgr.x on smithi179 2024-05-25T05:55:17.398 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-25T05:55:17.455 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_4 2024-05-25T05:55:17.511 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-25T05:55:17.511 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:17.511 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-05-25T05:55:17.511 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:17.511 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:17.511 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-05-25 05:54:52.901942133 +0000 2024-05-25T05:55:17.511 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-05-25 05:52:17.080437315 +0000 2024-05-25T05:55:17.512 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-05-25 05:52:17.080437315 +0000 2024-05-25T05:55:17.512 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-05-25 05:52:17.080437315 +0000 2024-05-25T05:55:17.512 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-25T05:55:17.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:17 smithi175 ceph-mon[24776]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi175=y;smithi179=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:17.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:17 smithi175 ceph-mon[24776]: Saving service mgr spec with placement smithi175=y;smithi179=x;count:2 2024-05-25T05:55:17.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:17 smithi175 ceph-mon[24776]: Deploying daemon mgr.x on smithi179 2024-05-25T05:55:17.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:17 smithi175 ceph-mon[29436]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi175=y;smithi179=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:17.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:17 smithi175 ceph-mon[29436]: Saving service mgr spec with placement smithi175=y;smithi179=x;count:2 2024-05-25T05:55:17.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:17 smithi175 ceph-mon[29436]: Deploying daemon mgr.x on smithi179 2024-05-25T05:55:17.575 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-05-25T05:55:17.575 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-05-25T05:55:17.575 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000187312 s, 2.7 MB/s 2024-05-25T05:55:17.576 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-25T05:55:17.632 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-05-25T05:55:17.633 DEBUG:teuthology.orchestra.run.smithi179:> dd if=/scratch_devs of=/dev/stdout 2024-05-25T05:55:17.649 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-25T05:55:17.649 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_1 2024-05-25T05:55:17.705 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-25T05:55:17.705 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:17.705 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-05-25T05:55:17.705 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:17.705 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:17.705 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-05-25 05:55:02.606944994 +0000 2024-05-25T05:55:17.706 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-05-25 05:52:22.677060762 +0000 2024-05-25T05:55:17.706 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-05-25 05:52:22.677060762 +0000 2024-05-25T05:55:17.706 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-05-25 05:52:22.677060762 +0000 2024-05-25T05:55:17.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-25T05:55:17.769 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-05-25T05:55:17.769 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-05-25T05:55:17.769 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000187425 s, 2.7 MB/s 2024-05-25T05:55:17.770 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-25T05:55:17.826 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_2 2024-05-25T05:55:17.881 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-05-25 05:55:02.607944971 +0000 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-05-25 05:52:22.998053087 +0000 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-05-25 05:52:22.998053087 +0000 2024-05-25T05:55:17.882 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-05-25 05:52:22.998053087 +0000 2024-05-25T05:55:17.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-25T05:55:17.944 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-05-25T05:55:17.944 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-05-25T05:55:17.944 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000173734 s, 2.9 MB/s 2024-05-25T05:55:17.945 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-25T05:55:18.001 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_3 2024-05-25T05:55:18.064 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-05-25 05:55:02.607944971 +0000 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-05-25 05:52:23.346044766 +0000 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-05-25 05:52:23.346044766 +0000 2024-05-25T05:55:18.065 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-05-25 05:52:23.346044766 +0000 2024-05-25T05:55:18.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-25T05:55:18.167 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-05-25T05:55:18.167 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-05-25T05:55:18.167 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000177352 s, 2.9 MB/s 2024-05-25T05:55:18.168 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-25T05:55:18.184 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_4 2024-05-25T05:55:18.243 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-25T05:55:18.243 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-25T05:55:18.243 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-05-25T05:55:18.243 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-25T05:55:18.244 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-05-25T05:55:18.244 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-05-25 05:55:18.125589121 +0000 2024-05-25T05:55:18.244 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-05-25 05:52:23.664037163 +0000 2024-05-25T05:55:18.244 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-05-25 05:52:23.664037163 +0000 2024-05-25T05:55:18.244 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-05-25 05:52:23.664037163 +0000 2024-05-25T05:55:18.244 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-25T05:55:18.340 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-05-25T05:55:18.341 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-05-25T05:55:18.341 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000194578 s, 2.6 MB/s 2024-05-25T05:55:18.342 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-25T05:55:18.357 INFO:tasks.cephadm:Deploying osd.0 on smithi175 with /dev/vg_nvme/lv_4... 2024-05-25T05:55:18.357 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-25T05:55:18.579 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:18 smithi179 ceph-mon[25623]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:18.579 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 systemd[1]: Starting Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T05:55:18.654 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:18 smithi175 ceph-mon[24776]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:18.654 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:18 smithi175 ceph-mon[29436]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:18.898 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 podman[26780]: 2024-05-25 05:55:18.481787448 +0000 UTC m=+0.017268524 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-25T05:55:18.898 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 podman[26780]: 2024-05-25 05:55:18.586861004 +0000 UTC m=+0.122342103 container create 31d6842a88f319a67eda3c1857487cd0c66677f1f18234e30c66233551550600 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, ceph=True, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, version=8, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-25T05:55:18.898 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 podman[26780]: 2024-05-25 05:55:18.79802234 +0000 UTC m=+0.333503411 container init 31d6842a88f319a67eda3c1857487cd0c66677f1f18234e30c66233551550600 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, ceph=True, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-type=git, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0) 2024-05-25T05:55:18.898 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 podman[26780]: 2024-05-25 05:55:18.800779981 +0000 UTC m=+0.336261065 container start 31d6842a88f319a67eda3c1857487cd0c66677f1f18234e30c66233551550600 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, name=centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, architecture=x86_64, version=8, io.buildah.version=1.19.8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754) 2024-05-25T05:55:18.898 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 ceph-mgr[26799]: -- 172.21.15.179:0/3396822180 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559cc0e4a340 con 0x559cc0e22c00 2024-05-25T05:55:18.898 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 bash[26780]: 31d6842a88f319a67eda3c1857487cd0c66677f1f18234e30c66233551550600 2024-05-25T05:55:19.186 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 systemd[1]: Started Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:55:19.186 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:18.944+0000 7f7721346000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T05:55:19.186 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:19.081+0000 7f7721346000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:55:19.186 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:19.185+0000 7f7721346000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T05:55:19.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:19.512+0000 7f7721346000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T05:55:19.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:19.599+0000 7f7721346000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T05:55:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:19 smithi179 ceph-mon[25623]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:19 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:19 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:19 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:19 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:20.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[24776]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:20.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:20.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:20.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:20.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:20.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[29436]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:20.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:20.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:20.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:20.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:19 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:20.365 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-25T05:55:20.365 INFO:teuthology.orchestra.run.smithi175.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-25T05:55:20.365 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10+0 records in 2024-05-25T05:55:20.365 INFO:teuthology.orchestra.run.smithi175.stderr:10+0 records out 2024-05-25T05:55:20.365 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113205 s, 926 MB/s 2024-05-25T05:55:20.365 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping successful for: 2024-05-25T05:55:21.011 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:20.751+0000 7f7721346000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T05:55:21.012 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:20.914+0000 7f7721346000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T05:55:21.012 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:20.995+0000 7f7721346000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T05:55:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:21 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:21 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:21 smithi179 ceph-mon[25623]: Reconfiguring mgr.y (unknown last config time)... 2024-05-25T05:55:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:21 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T05:55:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:21 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T05:55:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:21 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:21.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:21 smithi179 ceph-mon[25623]: Reconfiguring daemon mgr.y on smithi175 2024-05-25T05:55:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[24776]: Reconfiguring mgr.y (unknown last config time)... 2024-05-25T05:55:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T05:55:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T05:55:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[24776]: Reconfiguring daemon mgr.y on smithi175 2024-05-25T05:55:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[29436]: Reconfiguring mgr.y (unknown last config time)... 2024-05-25T05:55:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T05:55:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T05:55:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:21 smithi175 ceph-mon[29436]: Reconfiguring daemon mgr.y on smithi175 2024-05-25T05:55:21.730 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:21.308+0000 7f7721346000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T05:55:21.730 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:21.397+0000 7f7721346000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T05:55:21.731 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:21.489+0000 7f7721346000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T05:55:21.731 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:21.729+0000 7f7721346000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:55:21.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_4 2024-05-25T05:55:22.207 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[24776]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[29436]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:22.208 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:22.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:22.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:22.164+0000 7f7721346000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T05:55:22.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:22 smithi179 ceph-mon[25623]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:22.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:22.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:22.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:22.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:22.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:23.026 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:22.648+0000 7f7721346000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T05:55:23.027 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:22.741+0000 7f7721346000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T05:55:23.127 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:55:23.128 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:55:23.128 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:23.128 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:55:23.128 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:55:23.128 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:23.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:55:23.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:55:23.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:23.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:23.031+0000 7f7721346000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T05:55:23.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:23.120+0000 7f7721346000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T05:55:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:23.522+0000 7f7721346000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T05:55:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:23.652+0000 7f7721346000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T05:55:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:23.743+0000 7f7721346000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T05:55:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[24776]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:55:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:55:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[29436]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:55:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:55:24.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:24 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:24 smithi179 ceph-mon[25623]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:24 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:55:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:24 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:55:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:24 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:24.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:23.855+0000 7f7721346000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T05:55:24.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:23.942+0000 7f7721346000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T05:55:24.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:24.486+0000 7f7721346000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T05:55:25.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:55:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:55:24.846+0000 7f7721346000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T05:55:25.434 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[24776]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:25.434 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[24776]: Standby manager daemon x started 2024-05-25T05:55:25.434 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[29436]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T05:55:25.435 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:25 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T05:55:25.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:25 smithi179 ceph-mon[25623]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:25 smithi179 ceph-mon[25623]: Standby manager daemon x started 2024-05-25T05:55:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:25 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T05:55:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:25 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T05:55:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:25 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T05:55:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:25 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3154434223' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T05:55:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:26 smithi179 ceph-mon[25623]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:26 smithi179 ceph-mon[25623]: mgrmap e14: y(active, since 62s), standbys: x 2024-05-25T05:55:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:26 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T05:55:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:26 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2689250685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "127d9a00-0ee0-4842-bf83-11a1f4a93fe8"}]: dispatch 2024-05-25T05:55:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:26 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2689250685' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "127d9a00-0ee0-4842-bf83-11a1f4a93fe8"}]': finished 2024-05-25T05:55:26.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:26 smithi179 ceph-mon[25623]: osdmap e5: 1 total, 0 up, 1 in 2024-05-25T05:55:26.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:26 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[24776]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[24776]: mgrmap e14: y(active, since 62s), standbys: x 2024-05-25T05:55:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T05:55:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2689250685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "127d9a00-0ee0-4842-bf83-11a1f4a93fe8"}]: dispatch 2024-05-25T05:55:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2689250685' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "127d9a00-0ee0-4842-bf83-11a1f4a93fe8"}]': finished 2024-05-25T05:55:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[24776]: osdmap e5: 1 total, 0 up, 1 in 2024-05-25T05:55:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[29436]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[29436]: mgrmap e14: y(active, since 62s), standbys: x 2024-05-25T05:55:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T05:55:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2689250685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "127d9a00-0ee0-4842-bf83-11a1f4a93fe8"}]: dispatch 2024-05-25T05:55:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2689250685' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "127d9a00-0ee0-4842-bf83-11a1f4a93fe8"}]': finished 2024-05-25T05:55:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[29436]: osdmap e5: 1 total, 0 up, 1 in 2024-05-25T05:55:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:26 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:27.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:27 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1229307262' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:55:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:27 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1229307262' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:55:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:27 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1229307262' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:55:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:28 smithi179 ceph-mon[25623]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:28.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:28 smithi175 ceph-mon[24776]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:28 smithi175 ceph-mon[29436]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:30 smithi175 ceph-mon[24776]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:30 smithi175 ceph-mon[29436]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:30 smithi179 ceph-mon[25623]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:32.503 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:32 smithi175 ceph-mon[24776]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:32.504 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:32 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T05:55:32.504 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:32 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:32.504 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:32 smithi175 ceph-mon[29436]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:32.504 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:32 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T05:55:32.504 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:32 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:32.588 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:32 smithi179 ceph-mon[25623]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:32.588 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:32 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T05:55:32.588 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:32 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:33.543 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:33 smithi175 ceph-mon[24776]: Deploying daemon osd.0 on smithi175 2024-05-25T05:55:33.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:33 smithi175 ceph-mon[29436]: Deploying daemon osd.0 on smithi175 2024-05-25T05:55:33.600 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:33 smithi179 ceph-mon[25623]: Deploying daemon osd.0 on smithi175 2024-05-25T05:55:34.600 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:34 smithi175 ceph-mon[24776]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:34.601 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:34 smithi175 ceph-mon[29436]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:34.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:34 smithi179 ceph-mon[25623]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:36.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:36 smithi175 ceph-mon[24776]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:36.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:36 smithi175 ceph-mon[29436]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:36 smithi179 ceph-mon[25623]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:38 smithi179 ceph-mon[25623]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:38 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:38 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:38 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:38 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:38.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[29436]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:38.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:38.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:38.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:38.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:38.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[24776]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:38.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:38.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:38.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:38.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:38 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[29436]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[29436]: from='osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[29436]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[24776]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:40.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:40.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[24776]: from='osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T05:55:40.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:40 smithi175 ceph-mon[24776]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T05:55:40.619 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 0 on host 'smithi175' 2024-05-25T05:55:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:40 smithi179 ceph-mon[25623]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:40 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:40 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:40 smithi179 ceph-mon[25623]: from='osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T05:55:40.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:40 smithi179 ceph-mon[25623]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T05:55:41.280 DEBUG:teuthology.orchestra.run.smithi175:osd.0> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.0.service 2024-05-25T05:55:41.282 INFO:tasks.cephadm:Deploying osd.1 on smithi175 with /dev/vg_nvme/lv_3... 2024-05-25T05:55:41.283 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-25T05:55:41.542 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: osdmap e6: 1 total, 0 up, 1 in 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[29436]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:55:41.543 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: osdmap e6: 1 total, 0 up, 1 in 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:55:41.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:41 smithi175 ceph-mon[24776]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:55:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:55:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:55:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-25T05:55:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: osdmap e6: 1 total, 0 up, 1 in 2024-05-25T05:55:41.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:41.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:55:41.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:41 smithi179 ceph-mon[25623]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:55:42.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:55:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[32276]: 2024-05-25T05:55:42.007+0000 7fcb17918700 -1 osd.0 0 waiting for initial osdmap 2024-05-25T05:55:42.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:55:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[32276]: 2024-05-25T05:55:42.010+0000 7fcb122af700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:55:42.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[24776]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:42.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[24776]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:55:42.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[24776]: osdmap e7: 1 total, 0 up, 1 in 2024-05-25T05:55:42.584 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:42.585 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:42.585 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[29436]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:42.585 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[29436]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:55:42.585 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[29436]: osdmap e7: 1 total, 0 up, 1 in 2024-05-25T05:55:42.585 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:42.585 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:42 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:42 smithi179 ceph-mon[25623]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:42 smithi179 ceph-mon[25623]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:55:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:42 smithi179 ceph-mon[25623]: osdmap e7: 1 total, 0 up, 1 in 2024-05-25T05:55:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:42 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:42.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:42 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:44.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:55:44.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:55:44.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:44.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476] boot 2024-05-25T05:55:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: osdmap e8: 1 total, 1 up, 1 in 2024-05-25T05:55:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:55:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476] boot 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: osdmap e8: 1 total, 1 up, 1 in 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.294 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: osd.0 [v2:172.21.15.175:6802/2524900476,v1:172.21.15.175:6803/2524900476] boot 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: osdmap e8: 1 total, 1 up, 1 in 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:44.295 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:55:45.006 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-25T05:55:45.006 INFO:teuthology.orchestra.run.smithi175.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-25T05:55:45.006 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10+0 records in 2024-05-25T05:55:45.006 INFO:teuthology.orchestra.run.smithi175.stderr:10+0 records out 2024-05-25T05:55:45.006 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121285 s, 865 MB/s 2024-05-25T05:55:45.006 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping successful for: 2024-05-25T05:55:45.276 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:45 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi175 2024-05-25T05:55:45.276 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:45 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi175 to 16064M 2024-05-25T05:55:45.276 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:45 smithi175 ceph-mon[24776]: osdmap e9: 1 total, 1 up, 1 in 2024-05-25T05:55:45.277 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:45 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi175 2024-05-25T05:55:45.277 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:45 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi175 to 16064M 2024-05-25T05:55:45.277 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:45 smithi175 ceph-mon[29436]: osdmap e9: 1 total, 1 up, 1 in 2024-05-25T05:55:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:45 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi175 2024-05-25T05:55:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:45 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi175 to 16064M 2024-05-25T05:55:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:45 smithi179 ceph-mon[25623]: osdmap e9: 1 total, 1 up, 1 in 2024-05-25T05:55:45.692 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_3 2024-05-25T05:55:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:46 smithi175 ceph-mon[29436]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:46.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:46 smithi175 ceph-mon[24776]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:46 smithi179 ceph-mon[25623]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:48.551 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[24776]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:48.552 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:55:48.552 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:55:48.552 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:48.552 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[29436]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:48.552 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:55:48.552 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:55:48.552 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:48 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:48.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:48 smithi179 ceph-mon[25623]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:48.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:48 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:55:48.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:48 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:55:48.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:48 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:49.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:49 smithi175 ceph-mon[24776]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:49 smithi175 ceph-mon[29436]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:49.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:49 smithi179 ceph-mon[25623]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:55:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:50 smithi179 ceph-mon[25623]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:50 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1742081894' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]: dispatch 2024-05-25T05:55:50.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:50 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]: dispatch 2024-05-25T05:55:50.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:50 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]': finished 2024-05-25T05:55:50.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:50 smithi179 ceph-mon[25623]: osdmap e10: 2 total, 1 up, 2 in 2024-05-25T05:55:50.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:50 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:55:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[24776]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1742081894' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]: dispatch 2024-05-25T05:55:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]: dispatch 2024-05-25T05:55:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]': finished 2024-05-25T05:55:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[24776]: osdmap e10: 2 total, 1 up, 2 in 2024-05-25T05:55:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:55:50.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[29436]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:50.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1742081894' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]: dispatch 2024-05-25T05:55:50.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]: dispatch 2024-05-25T05:55:50.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1a2ed55d-9db0-4df8-be29-d0a3313205ab"}]': finished 2024-05-25T05:55:50.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[29436]: osdmap e10: 2 total, 1 up, 2 in 2024-05-25T05:55:50.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:50 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:55:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:51 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2335340518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:55:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:51 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2335340518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:55:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:51 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2335340518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:55:52.605 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:52 smithi175 ceph-mon[29436]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:52.606 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:52 smithi175 ceph-mon[24776]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:52 smithi179 ceph-mon[25623]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:54.483 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:54 smithi175 ceph-mon[24776]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:54.484 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:54 smithi175 ceph-mon[29436]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:54 smithi179 ceph-mon[25623]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:56.597 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:56 smithi175 ceph-mon[24776]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:56.597 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T05:55:56.597 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:56.597 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:56 smithi175 ceph-mon[29436]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:56.597 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T05:55:56.597 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:56 smithi179 ceph-mon[25623]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T05:55:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:55:57.456 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:57 smithi175 ceph-mon[24776]: Deploying daemon osd.1 on smithi175 2024-05-25T05:55:57.456 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:57 smithi175 ceph-mon[29436]: Deploying daemon osd.1 on smithi175 2024-05-25T05:55:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:57 smithi179 ceph-mon[25623]: Deploying daemon osd.1 on smithi175 2024-05-25T05:55:58.721 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:55:58 smithi175 ceph-mon[24776]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:58.721 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:55:58 smithi175 ceph-mon[29436]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:55:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:55:58 smithi179 ceph-mon[25623]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:00.579 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:00 smithi175 ceph-mon[24776]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:00.579 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:00 smithi175 ceph-mon[29436]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:00 smithi179 ceph-mon[25623]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:02.372 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[29436]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:02.372 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[24776]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:02.373 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:02 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:02.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:02 smithi179 ceph-mon[25623]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:02.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:02 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:02.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:02 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:02.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:02 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:02.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:02 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:04.156 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 1 on host 'smithi175' 2024-05-25T05:56:04.422 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:04 smithi175 ceph-mon[24776]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:04.422 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:04 smithi175 ceph-mon[29436]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:04.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:04 smithi179 ceph-mon[25623]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:04.811 DEBUG:teuthology.orchestra.run.smithi175:osd.1> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.1.service 2024-05-25T05:56:04.813 INFO:tasks.cephadm:Deploying osd.2 on smithi175 with /dev/vg_nvme/lv_2... 2024-05-25T05:56:04.813 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-25T05:56:05.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:05.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[24776]: from='osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T05:56:05.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[24776]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T05:56:05.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:05.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:05.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[29436]: from='osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T05:56:05.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[29436]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T05:56:05.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:05 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:05 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:05 smithi179 ceph-mon[25623]: from='osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T05:56:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:05 smithi179 ceph-mon[25623]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T05:56:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:05 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:06.522 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[29436]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:06.522 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[29436]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-25T05:56:06.522 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[29436]: osdmap e11: 2 total, 1 up, 2 in 2024-05-25T05:56:06.522 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[29436]: from='osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[29436]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[24776]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[24776]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[24776]: osdmap e11: 2 total, 1 up, 2 in 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[24776]: from='osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:06.523 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:06 smithi175 ceph-mon[24776]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:06.524 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[35225]: 2024-05-25T05:56:06.172+0000 7f598dc11700 -1 osd.1 0 waiting for initial osdmap 2024-05-25T05:56:06.524 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[35225]: 2024-05-25T05:56:06.175+0000 7f598a5ac700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:56:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:06 smithi179 ceph-mon[25623]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:06 smithi179 ceph-mon[25623]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-25T05:56:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:06 smithi179 ceph-mon[25623]: osdmap e11: 2 total, 1 up, 2 in 2024-05-25T05:56:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:06 smithi179 ceph-mon[25623]: from='osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:06 smithi179 ceph-mon[25623]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:07.378 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[24776]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:07.378 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[24776]: osdmap e12: 2 total, 1 up, 2 in 2024-05-25T05:56:07.378 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:07.379 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:07.379 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[29436]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:07.379 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[29436]: osdmap e12: 2 total, 1 up, 2 in 2024-05-25T05:56:07.379 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:07.379 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:07 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:07 smithi179 ceph-mon[25623]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:07 smithi179 ceph-mon[25623]: osdmap e12: 2 total, 1 up, 2 in 2024-05-25T05:56:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:07 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:07 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:08.123 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-25T05:56:08.123 INFO:teuthology.orchestra.run.smithi175.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-25T05:56:08.123 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10+0 records in 2024-05-25T05:56:08.123 INFO:teuthology.orchestra.run.smithi175.stderr:10+0 records out 2024-05-25T05:56:08.123 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124952 s, 839 MB/s 2024-05-25T05:56:08.123 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping successful for: 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[24776]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[24776]: osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965] boot 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[24776]: osdmap e13: 2 total, 2 up, 2 in 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:08.419 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:56:08.420 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:56:08.420 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[29436]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:08.420 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:08.420 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[29436]: osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965] boot 2024-05-25T05:56:08.420 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[29436]: osdmap e13: 2 total, 2 up, 2 in 2024-05-25T05:56:08.420 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:08 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:56:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:08 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:56:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:08 smithi179 ceph-mon[25623]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-25T05:56:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:08.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:08 smithi179 ceph-mon[25623]: osd.1 [v2:172.21.15.175:6810/417501965,v1:172.21.15.175:6811/417501965] boot 2024-05-25T05:56:08.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:08 smithi179 ceph-mon[25623]: osdmap e13: 2 total, 2 up, 2 in 2024-05-25T05:56:08.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:56:09.096 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_2 2024-05-25T05:56:09.403 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: osdmap e14: 2 total, 2 up, 2 in 2024-05-25T05:56:09.403 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.403 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:09.403 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: osdmap e14: 2 total, 2 up, 2 in 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:09.404 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.405 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.405 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:09.405 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:09.405 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:09.405 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:09 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: osdmap e14: 2 total, 2 up, 2 in 2024-05-25T05:56:09.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:09.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:09 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:10.593 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:10 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi175 2024-05-25T05:56:10.593 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:10 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi175 to 8032M 2024-05-25T05:56:10.593 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:10 smithi175 ceph-mon[29436]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:10.594 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:10 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi175 2024-05-25T05:56:10.594 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:10 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi175 to 8032M 2024-05-25T05:56:10.594 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:10 smithi175 ceph-mon[24776]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:10 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi175 2024-05-25T05:56:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:10 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi175 to 8032M 2024-05-25T05:56:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:10 smithi179 ceph-mon[25623]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:11.652 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:11 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:11.652 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:11 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:11.652 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:11 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:11.652 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:11 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:11.652 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:11 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:11.652 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:11 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:11 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:11 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:11 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:12.540 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:12 smithi175 ceph-mon[29436]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:12.541 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:12 smithi175 ceph-mon[29436]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:12.541 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:12 smithi175 ceph-mon[24776]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:12.541 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:12 smithi175 ceph-mon[24776]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:12.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:12 smithi179 ceph-mon[25623]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:12.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:12 smithi179 ceph-mon[25623]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:13 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3203245943' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]: dispatch 2024-05-25T05:56:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:13 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]: dispatch 2024-05-25T05:56:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:13 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]': finished 2024-05-25T05:56:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:13 smithi179 ceph-mon[25623]: osdmap e15: 3 total, 2 up, 3 in 2024-05-25T05:56:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:13 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3203245943' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]: dispatch 2024-05-25T05:56:13.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]: dispatch 2024-05-25T05:56:13.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]': finished 2024-05-25T05:56:13.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[29436]: osdmap e15: 3 total, 2 up, 3 in 2024-05-25T05:56:13.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:13.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3203245943' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]: dispatch 2024-05-25T05:56:13.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]: dispatch 2024-05-25T05:56:13.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fad27f8-b98f-43f6-a52e-f64ed125afd7"}]': finished 2024-05-25T05:56:13.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[24776]: osdmap e15: 3 total, 2 up, 3 in 2024-05-25T05:56:13.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:13 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:14 smithi179 ceph-mon[25623]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:14 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2139801361' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:56:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:14 smithi175 ceph-mon[24776]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:14 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2139801361' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:56:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:14 smithi175 ceph-mon[29436]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:14 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2139801361' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:56:16.486 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:16 smithi175 ceph-mon[24776]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:16.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:16 smithi175 ceph-mon[29436]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:16 smithi179 ceph-mon[25623]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:18.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:18 smithi175 ceph-mon[24776]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:18.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:18 smithi175 ceph-mon[29436]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:18 smithi179 ceph-mon[25623]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:20 smithi175 ceph-mon[29436]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:20 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T05:56:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:20 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:20.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:20 smithi175 ceph-mon[24776]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:20.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:20 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T05:56:20.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:20 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:20 smithi179 ceph-mon[25623]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:20 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T05:56:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:20 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:21.750 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:21 smithi175 ceph-mon[29436]: Deploying daemon osd.2 on smithi175 2024-05-25T05:56:21.751 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:21 smithi175 ceph-mon[24776]: Deploying daemon osd.2 on smithi175 2024-05-25T05:56:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:21 smithi179 ceph-mon[25623]: Deploying daemon osd.2 on smithi175 2024-05-25T05:56:22.643 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:22 smithi175 ceph-mon[24776]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:22.644 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:22 smithi175 ceph-mon[29436]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:22 smithi179 ceph-mon[25623]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:23.709 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:56:23.709 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:56:23.710 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:56:23.710 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:56:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:56:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:56:24.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:24 smithi175 ceph-mon[24776]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:24.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:24 smithi175 ceph-mon[29436]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:24 smithi179 ceph-mon[25623]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:25 smithi179 ceph-mon[25623]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:26.042 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[24776]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[29436]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:26.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:26.044 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:27.695 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 2 on host 'smithi175' 2024-05-25T05:56:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:27 smithi179 ceph-mon[25623]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:28.038 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:27 smithi175 ceph-mon[24776]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:28.038 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:27 smithi175 ceph-mon[29436]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:28.664 DEBUG:teuthology.orchestra.run.smithi175:osd.2> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.2.service 2024-05-25T05:56:28.667 INFO:tasks.cephadm:Deploying osd.3 on smithi175 with /dev/vg_nvme/lv_1... 2024-05-25T05:56:28.667 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-25T05:56:28.894 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:28.895 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:28 smithi175 ceph-mon[24776]: from='osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T05:56:28.895 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:28 smithi175 ceph-mon[24776]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T05:56:28.895 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:28.895 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:28 smithi175 ceph-mon[29436]: from='osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T05:56:28.895 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:28 smithi175 ceph-mon[29436]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T05:56:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:28 smithi179 ceph-mon[25623]: from='osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T05:56:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:28 smithi179 ceph-mon[25623]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T05:56:29.989 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[29436]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:29.989 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[29436]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-25T05:56:29.989 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[29436]: osdmap e16: 3 total, 2 up, 3 in 2024-05-25T05:56:29.989 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:29.990 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[29436]: from='osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:29.990 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[29436]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:29.990 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:29.990 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[38128]: 2024-05-25T05:56:29.707+0000 7f4dc2705700 -1 osd.2 0 waiting for initial osdmap 2024-05-25T05:56:29.990 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[38128]: 2024-05-25T05:56:29.709+0000 7f4dbd09c700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:56:29.990 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[24776]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:29.990 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[24776]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-25T05:56:29.991 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[24776]: osdmap e16: 3 total, 2 up, 3 in 2024-05-25T05:56:29.991 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:29.991 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[24776]: from='osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:29.991 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[24776]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:29.991 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:29 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:30.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:29 smithi179 ceph-mon[25623]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:30.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:29 smithi179 ceph-mon[25623]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-25T05:56:30.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:29 smithi179 ceph-mon[25623]: osdmap e16: 3 total, 2 up, 3 in 2024-05-25T05:56:30.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:29 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:30.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:29 smithi179 ceph-mon[25623]: from='osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:30.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:29 smithi179 ceph-mon[25623]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:30.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:29 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[24776]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[24776]: osdmap e17: 3 total, 2 up, 3 in 2024-05-25T05:56:30.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:30.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:30.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[29436]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:30.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[29436]: osdmap e17: 3 total, 2 up, 3 in 2024-05-25T05:56:30.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:30.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:30 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:30.993 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-25T05:56:30.993 INFO:teuthology.orchestra.run.smithi175.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-25T05:56:30.993 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10+0 records in 2024-05-25T05:56:30.993 INFO:teuthology.orchestra.run.smithi175.stderr:10+0 records out 2024-05-25T05:56:30.993 INFO:teuthology.orchestra.run.smithi175.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122698 s, 855 MB/s 2024-05-25T05:56:30.993 INFO:teuthology.orchestra.run.smithi175.stderr:--> Zapping successful for: 2024-05-25T05:56:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:30 smithi179 ceph-mon[25623]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:30 smithi179 ceph-mon[25623]: osdmap e17: 3 total, 2 up, 3 in 2024-05-25T05:56:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:30 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:30 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:31.723 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_1 2024-05-25T05:56:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:31 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:56:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:31 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:56:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:31 smithi179 ceph-mon[25623]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:31 smithi179 ceph-mon[25623]: osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520] boot 2024-05-25T05:56:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:31 smithi179 ceph-mon[25623]: osdmap e18: 3 total, 3 up, 3 in 2024-05-25T05:56:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:31 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:56:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:56:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[24776]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[24776]: osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520] boot 2024-05-25T05:56:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[24776]: osdmap e18: 3 total, 3 up, 3 in 2024-05-25T05:56:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:56:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:56:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[29436]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-25T05:56:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[29436]: osd.2 [v2:172.21.15.175:6818/4111967520,v1:172.21.15.175:6819/4111967520] boot 2024-05-25T05:56:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[29436]: osdmap e18: 3 total, 3 up, 3 in 2024-05-25T05:56:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:31 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:56:32.975 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:32 smithi175 ceph-mon[24776]: osdmap e19: 3 total, 3 up, 3 in 2024-05-25T05:56:32.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:32 smithi175 ceph-mon[29436]: osdmap e19: 3 total, 3 up, 3 in 2024-05-25T05:56:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:32 smithi179 ceph-mon[25623]: osdmap e19: 3 total, 3 up, 3 in 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: osdmap e20: 3 total, 3 up, 3 in 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-25T05:56:34.242 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: osdmap e20: 3 total, 3 up, 3 in 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:34.243 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:33 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-25T05:56:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-25T05:56:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: osdmap e20: 3 total, 3 up, 3 in 2024-05-25T05:56:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:33 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:34.772 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:56:34 smithi175 sudo[39430]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-25T05:56:34.772 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:56:34 smithi175 sudo[39430]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:34.772 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:56:34 smithi175 sudo[39430]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:35.050 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:56:34 smithi175 sudo[39458]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-25T05:56:35.050 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:56:34 smithi175 sudo[39458]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:35.050 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 05:56:34 smithi175 sudo[39458]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi175 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi175 to 5354M 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: osdmap e21: 3 total, 3 up, 3 in 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi175 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi175 to 5354M 2024-05-25T05:56:35.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-25T05:56:35.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: osdmap e21: 3 total, 3 up, 3 in 2024-05-25T05:56:35.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:35.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:35.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:35.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi175 2024-05-25T05:56:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi175 to 5354M 2024-05-25T05:56:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-25T05:56:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: osdmap e21: 3 total, 3 up, 3 in 2024-05-25T05:56:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:35.303 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39498]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-25T05:56:35.303 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39498]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:35.303 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39498]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:35.562 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39542]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-25T05:56:35.563 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39542]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:35.563 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39542]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:35.814 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39563]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-25T05:56:35.814 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39563]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:35.814 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39563]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:36.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39566]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-25T05:56:36.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39566]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:36.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 05:56:35 smithi175 sudo[39566]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39570]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-25T05:56:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39570]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:36.387 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[29436]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:36.387 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[29436]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:36.387 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[29436]: osdmap e22: 3 total, 3 up, 3 in 2024-05-25T05:56:36.387 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[29436]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-25T05:56:36.387 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[29436]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39570]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39594]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39594]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39594]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[24776]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[24776]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[24776]: osdmap e22: 3 total, 3 up, 3 in 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[24776]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-25T05:56:36.388 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:36 smithi175 ceph-mon[24776]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:36.688 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39601]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-25T05:56:36.688 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39601]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:36.688 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39601]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:36 smithi179 ceph-mon[25623]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:36 smithi179 ceph-mon[25623]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:36 smithi179 ceph-mon[25623]: osdmap e22: 3 total, 3 up, 3 in 2024-05-25T05:56:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:36 smithi179 ceph-mon[25623]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-25T05:56:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:36 smithi179 ceph-mon[25623]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39606]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-25T05:56:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39606]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:36 smithi175 sudo[39606]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:37.324 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 sudo[27130]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-25T05:56:37.324 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 sudo[27130]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:37.324 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 sudo[27130]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 sudo[27133]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-25T05:56:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 sudo[27133]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-25T05:56:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 sudo[27133]: pam_unix(sudo:session): session closed for user root 2024-05-25T05:56:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/912338798' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]: dispatch 2024-05-25T05:56:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:37 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/912338798' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:37.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:56:37.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:56:37.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:56:37.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/912338798' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]: dispatch 2024-05-25T05:56:37.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:37 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]: dispatch 2024-05-25T05:56:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]': finished 2024-05-25T05:56:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: osdmap e23: 4 total, 3 up, 4 in 2024-05-25T05:56:38.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:38.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:38.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:38 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/132641461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]': finished 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: osdmap e23: 4 total, 3 up, 4 in 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/132641461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:56:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:38.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-25T05:56:38.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-25T05:56:38.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48140d40-4271-49e8-9e16-afb39b0de88f"}]': finished 2024-05-25T05:56:38.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: osdmap e23: 4 total, 3 up, 4 in 2024-05-25T05:56:38.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:38.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:38.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:38 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/132641461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:56:40.450 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:40 smithi175 ceph-mon[29436]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:40.450 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:40 smithi175 ceph-mon[24776]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:40 smithi179 ceph-mon[25623]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:42.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:42 smithi175 ceph-mon[24776]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:42.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:42 smithi175 ceph-mon[29436]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:42 smithi179 ceph-mon[25623]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[24776]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T05:56:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[24776]: Deploying daemon osd.3 on smithi175 2024-05-25T05:56:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[29436]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T05:56:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:44.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:44 smithi175 ceph-mon[29436]: Deploying daemon osd.3 on smithi175 2024-05-25T05:56:44.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:44 smithi179 ceph-mon[25623]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:44.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T05:56:44.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:44.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:44 smithi179 ceph-mon[25623]: Deploying daemon osd.3 on smithi175 2024-05-25T05:56:46.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:46 smithi175 ceph-mon[24776]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:46.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:46 smithi175 ceph-mon[29436]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:46 smithi179 ceph-mon[25623]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:48.541 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:48 smithi175 ceph-mon[24776]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:48.541 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:48 smithi175 ceph-mon[29436]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:48.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:48 smithi179 ceph-mon[25623]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[24776]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:49.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:49.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:49.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:49.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:49.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:49 smithi175 ceph-mon[29436]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:49 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:49 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:49 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:49 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:49 smithi179 ceph-mon[25623]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:51.706 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 3 on host 'smithi175' 2024-05-25T05:56:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:51 smithi179 ceph-mon[25623]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:51 smithi179 ceph-mon[25623]: from='osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T05:56:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:51 smithi179 ceph-mon[25623]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T05:56:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:51 smithi175 ceph-mon[24776]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:51 smithi175 ceph-mon[24776]: from='osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T05:56:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:51 smithi175 ceph-mon[24776]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T05:56:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:51 smithi175 ceph-mon[29436]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:51 smithi175 ceph-mon[29436]: from='osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T05:56:52.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:51 smithi175 ceph-mon[29436]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T05:56:52.343 DEBUG:teuthology.orchestra.run.smithi175:osd.3> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.3.service 2024-05-25T05:56:52.345 INFO:tasks.cephadm:Deploying osd.4 on smithi179 with /dev/vg_nvme/lv_4... 2024-05-25T05:56:52.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-25T05:56:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-25T05:56:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: osdmap e24: 4 total, 3 up, 4 in 2024-05-25T05:56:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: osdmap e25: 4 total, 3 up, 4 in 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: osdmap e24: 4 total, 3 up, 4 in 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: osdmap e25: 4 total, 3 up, 4 in 2024-05-25T05:56:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:52 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:52.822 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 05:56:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[41081]: 2024-05-25T05:56:52.689+0000 7ff3e6122700 -1 osd.3 0 waiting for initial osdmap 2024-05-25T05:56:52.822 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 05:56:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[41081]: 2024-05-25T05:56:52.694+0000 7ff3deab5700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:56:52.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-25T05:56:52.969 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: osdmap e24: 4 total, 3 up, 4 in 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: osdmap e25: 4 total, 3 up, 4 in 2024-05-25T05:56:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:52 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:53 smithi175 ceph-mon[29436]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:53 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:53 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:53 smithi175 ceph-mon[24776]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:53 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:53.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:53 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:53.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:53 smithi179 ceph-mon[25623]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:53.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:53 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:53.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:53 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:54.298 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-25T05:56:54.298 INFO:teuthology.orchestra.run.smithi179.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-25T05:56:54.298 INFO:teuthology.orchestra.run.smithi179.stderr: stderr: 10+0 records in 2024-05-25T05:56:54.298 INFO:teuthology.orchestra.run.smithi179.stderr:10+0 records out 2024-05-25T05:56:54.298 INFO:teuthology.orchestra.run.smithi179.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0118687 s, 883 MB/s 2024-05-25T05:56:54.298 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping successful for: 2024-05-25T05:56:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:56:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:56:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[29436]: osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094] boot 2024-05-25T05:56:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[29436]: osdmap e26: 4 total, 4 up, 4 in 2024-05-25T05:56:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:56:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:56:54.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[24776]: osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094] boot 2024-05-25T05:56:54.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[24776]: osdmap e26: 4 total, 4 up, 4 in 2024-05-25T05:56:54.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:54 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:54.894 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_4 2024-05-25T05:56:54.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:54 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:56:54.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:54 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:56:54.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:54 smithi179 ceph-mon[25623]: osd.3 [v2:172.21.15.175:6826/1737862094,v1:172.21.15.175:6827/1737862094] boot 2024-05-25T05:56:54.918 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:54 smithi179 ceph-mon[25623]: osdmap e26: 4 total, 4 up, 4 in 2024-05-25T05:56:54.918 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:54 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: osdmap e27: 4 total, 4 up, 4 in 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.375 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.376 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:56.376 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:56.376 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:56.376 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:56 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: osdmap e27: 4 total, 4 up, 4 in 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: osdmap e27: 4 total, 4 up, 4 in 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:56:56.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:56.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:56:56.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:56.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:56:56.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:56 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:56:57.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:57 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi175 2024-05-25T05:56:57.416 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:57 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi175 to 4016M 2024-05-25T05:56:57.416 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:57 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:57.416 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:57 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:57.416 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:57 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi175 2024-05-25T05:56:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi175 to 4016M 2024-05-25T05:56:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi175 2024-05-25T05:56:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi175 to 4016M 2024-05-25T05:56:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:56:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:56:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:57 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:56:58.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:58 smithi179 ceph-mon[25623]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:56:58.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:58 smithi179 ceph-mon[25623]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:58 smithi175 ceph-mon[29436]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:56:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:58 smithi175 ceph-mon[29436]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:58 smithi175 ceph-mon[24776]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:56:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:58 smithi175 ceph-mon[24776]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:56:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:59 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/4209329258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]: dispatch 2024-05-25T05:56:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:59 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]: dispatch 2024-05-25T05:56:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:59 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]': finished 2024-05-25T05:56:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:59 smithi179 ceph-mon[25623]: osdmap e28: 5 total, 4 up, 5 in 2024-05-25T05:56:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:56:59 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:56:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/4209329258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]: dispatch 2024-05-25T05:56:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]: dispatch 2024-05-25T05:56:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]': finished 2024-05-25T05:56:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[29436]: osdmap e28: 5 total, 4 up, 5 in 2024-05-25T05:56:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:56:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/4209329258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]: dispatch 2024-05-25T05:56:59.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]: dispatch 2024-05-25T05:56:59.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a65572db-45a9-47ee-a565-079c774a48f5"}]': finished 2024-05-25T05:56:59.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[24776]: osdmap e28: 5 total, 4 up, 5 in 2024-05-25T05:56:59.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:56:59 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:00 smithi179 ceph-mon[25623]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:00 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/1419720943' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:00 smithi175 ceph-mon[29436]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:00 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/1419720943' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:00 smithi175 ceph-mon[24776]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:00 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/1419720943' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:02.708 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:02 smithi179 ceph-mon[25623]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:02 smithi175 ceph-mon[29436]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:02 smithi175 ceph-mon[24776]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:04.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:04 smithi179 ceph-mon[25623]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:04 smithi175 ceph-mon[29436]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:04 smithi175 ceph-mon[24776]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:05.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:05 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T05:57:05.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:05 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:05 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T05:57:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:05 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:05 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T05:57:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:05 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:06.530 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:06 smithi179 ceph-mon[25623]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:06.530 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:06 smithi179 ceph-mon[25623]: Deploying daemon osd.4 on smithi179 2024-05-25T05:57:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:06 smithi175 ceph-mon[24776]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:06 smithi175 ceph-mon[24776]: Deploying daemon osd.4 on smithi179 2024-05-25T05:57:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:06 smithi175 ceph-mon[29436]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:06 smithi175 ceph-mon[29436]: Deploying daemon osd.4 on smithi179 2024-05-25T05:57:08.472 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:08 smithi179 ceph-mon[25623]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:08 smithi175 ceph-mon[24776]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:08 smithi175 ceph-mon[29436]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:10 smithi179 ceph-mon[25623]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:10.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[24776]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[29436]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:10.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:10.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:10.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:10.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:12 smithi179 ceph-mon[25623]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:12 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:12.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:12 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:12 smithi175 ceph-mon[24776]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:12 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:12 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:12 smithi175 ceph-mon[29436]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:12 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:12 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:13.377 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 4 on host 'smithi179' 2024-05-25T05:57:13.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:13 smithi179 ceph-mon[25623]: from='osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T05:57:13.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:13 smithi179 ceph-mon[25623]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T05:57:13.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:13 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:13 smithi175 ceph-mon[24776]: from='osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T05:57:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:13 smithi175 ceph-mon[24776]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T05:57:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:13 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:13 smithi175 ceph-mon[29436]: from='osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T05:57:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:13 smithi175 ceph-mon[29436]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T05:57:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:13 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:14.048 DEBUG:teuthology.orchestra.run.smithi179:osd.4> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.4.service 2024-05-25T05:57:14.091 INFO:tasks.cephadm:Deploying osd.5 on smithi179 with /dev/vg_nvme/lv_3... 2024-05-25T05:57:14.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-25T05:57:14.629 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:14.629 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:14.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:14.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:14.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-25T05:57:14.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: osdmap e29: 5 total, 4 up, 5 in 2024-05-25T05:57:14.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:14.630 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: from='osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:14.631 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:14 smithi179 ceph-mon[25623]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:14.631 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 05:57:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[28968]: 2024-05-25T05:57:14.403+0000 7fef62e84700 -1 osd.4 0 waiting for initial osdmap 2024-05-25T05:57:14.631 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 05:57:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[28968]: 2024-05-25T05:57:14.408+0000 7fef5f81f700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:57:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-25T05:57:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: osdmap e29: 5 total, 4 up, 5 in 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: from='osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[24776]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:14.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-25T05:57:14.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: osdmap e29: 5 total, 4 up, 5 in 2024-05-25T05:57:14.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:14.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: from='osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:14.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:14 smithi175 ceph-mon[29436]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:15.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:15 smithi179 ceph-mon[25623]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:57:15.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:15 smithi179 ceph-mon[25623]: osdmap e30: 5 total, 4 up, 5 in 2024-05-25T05:57:15.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:15 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:15 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:15 smithi179 ceph-mon[25623]: osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389] boot 2024-05-25T05:57:15.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:15 smithi179 ceph-mon[25623]: osdmap e31: 5 total, 5 up, 5 in 2024-05-25T05:57:15.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:15 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[24776]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:57:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[24776]: osdmap e30: 5 total, 4 up, 5 in 2024-05-25T05:57:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[24776]: osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389] boot 2024-05-25T05:57:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[24776]: osdmap e31: 5 total, 5 up, 5 in 2024-05-25T05:57:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[29436]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:57:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[29436]: osdmap e30: 5 total, 4 up, 5 in 2024-05-25T05:57:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[29436]: osd.4 [v2:172.21.15.179:6800/4035633389,v1:172.21.15.179:6801/4035633389] boot 2024-05-25T05:57:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[29436]: osdmap e31: 5 total, 5 up, 5 in 2024-05-25T05:57:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:15 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:57:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:16 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:57:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:16 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:57:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:16 smithi179 ceph-mon[25623]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:16 smithi179 ceph-mon[25623]: osdmap e32: 5 total, 5 up, 5 in 2024-05-25T05:57:16.721 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:57:16.722 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:57:16.722 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[29436]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:16.722 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[29436]: osdmap e32: 5 total, 5 up, 5 in 2024-05-25T05:57:16.722 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:57:16.722 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:57:16.722 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[24776]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-25T05:57:16.722 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:16 smithi175 ceph-mon[24776]: osdmap e32: 5 total, 5 up, 5 in 2024-05-25T05:57:17.610 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-25T05:57:17.610 INFO:teuthology.orchestra.run.smithi179.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-25T05:57:17.610 INFO:teuthology.orchestra.run.smithi179.stderr: stderr: 10+0 records in 2024-05-25T05:57:17.610 INFO:teuthology.orchestra.run.smithi179.stderr:10+0 records out 2024-05-25T05:57:17.610 INFO:teuthology.orchestra.run.smithi179.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111541 s, 940 MB/s 2024-05-25T05:57:17.610 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping successful for: 2024-05-25T05:57:18.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_3 2024-05-25T05:57:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:18 smithi179 ceph-mon[25623]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:18 smithi179 ceph-mon[25623]: osdmap e33: 5 total, 5 up, 5 in 2024-05-25T05:57:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:18 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:18 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:18 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:18 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[29436]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:18.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[29436]: osdmap e33: 5 total, 5 up, 5 in 2024-05-25T05:57:18.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:18.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[24776]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:18.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[24776]: osdmap e33: 5 total, 5 up, 5 in 2024-05-25T05:57:18.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:18.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:18.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:18 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:19 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi179 2024-05-25T05:57:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:19 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi179 to 17088M 2024-05-25T05:57:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:19 smithi179 ceph-mon[25623]: osdmap e34: 5 total, 5 up, 5 in 2024-05-25T05:57:19.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:19 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi179 2024-05-25T05:57:19.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:19 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi179 to 17088M 2024-05-25T05:57:19.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:19 smithi175 ceph-mon[24776]: osdmap e34: 5 total, 5 up, 5 in 2024-05-25T05:57:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:19 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi179 2024-05-25T05:57:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:19 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi179 to 17088M 2024-05-25T05:57:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:19 smithi175 ceph-mon[29436]: osdmap e34: 5 total, 5 up, 5 in 2024-05-25T05:57:20.491 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:20 smithi179 ceph-mon[25623]: pgmap v112: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-25T05:57:20.492 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:20 smithi179 ceph-mon[25623]: osdmap e35: 5 total, 5 up, 5 in 2024-05-25T05:57:20.492 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:20 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:57:20.492 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:20 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:57:20.492 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:20 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[24776]: pgmap v112: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-25T05:57:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[24776]: osdmap e35: 5 total, 5 up, 5 in 2024-05-25T05:57:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:57:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:57:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[29436]: pgmap v112: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-25T05:57:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[29436]: osdmap e35: 5 total, 5 up, 5 in 2024-05-25T05:57:20.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:57:20.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:57:20.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:20 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:21.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:21 smithi179 ceph-mon[25623]: from='client.24227 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:57:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:21 smithi175 ceph-mon[24776]: from='client.24227 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:57:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:21 smithi175 ceph-mon[29436]: from='client.24227 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:57:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:22 smithi179 ceph-mon[25623]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 122 KiB/s, 0 objects/s recovering 2024-05-25T05:57:22.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:22 smithi175 ceph-mon[29436]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 122 KiB/s, 0 objects/s recovering 2024-05-25T05:57:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:22 smithi175 ceph-mon[24776]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 122 KiB/s, 0 objects/s recovering 2024-05-25T05:57:22.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:57:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:57:22.617+0000 7f2f42e13700 -1 calc_pg_upmaps abort due to max <= 0 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/1400527246' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]: dispatch 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]: dispatch 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]': finished 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: osdmap e36: 6 total, 5 up, 6 in 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:57:23.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:23 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/2835004854' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/1400527246' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]: dispatch 2024-05-25T05:57:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]: dispatch 2024-05-25T05:57:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]': finished 2024-05-25T05:57:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: osdmap e36: 6 total, 5 up, 6 in 2024-05-25T05:57:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/2835004854' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/1400527246' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8b21121-8638-48a5-884e-93be0d0d8c17"}]': finished 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: osdmap e36: 6 total, 5 up, 6 in 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:57:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:23 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/2835004854' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:24 smithi179 ceph-mon[25623]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-05-25T05:57:24.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:24 smithi175 ceph-mon[29436]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-05-25T05:57:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:24 smithi175 ceph-mon[24776]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-05-25T05:57:26.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:26 smithi179 ceph-mon[25623]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:26 smithi175 ceph-mon[29436]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:26 smithi175 ceph-mon[24776]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:28.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:28 smithi179 ceph-mon[25623]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:28 smithi175 ceph-mon[24776]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:28 smithi175 ceph-mon[29436]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:29.758 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:29 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T05:57:29.758 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:29 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:29 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T05:57:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:29 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:29 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T05:57:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:29 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:30 smithi179 ceph-mon[25623]: Deploying daemon osd.5 on smithi179 2024-05-25T05:57:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:30 smithi179 ceph-mon[25623]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:30 smithi175 ceph-mon[24776]: Deploying daemon osd.5 on smithi179 2024-05-25T05:57:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:30 smithi175 ceph-mon[24776]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:30 smithi175 ceph-mon[29436]: Deploying daemon osd.5 on smithi179 2024-05-25T05:57:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:30 smithi175 ceph-mon[29436]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:32.734 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:32 smithi179 ceph-mon[25623]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:32.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:32 smithi175 ceph-mon[29436]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:32 smithi175 ceph-mon[24776]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:34 smithi179 ceph-mon[25623]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:34 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[24776]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[29436]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:34.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:34.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:34.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:34 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:36 smithi179 ceph-mon[25623]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:36 smithi175 ceph-mon[24776]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:36 smithi175 ceph-mon[29436]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 5 on host 'smithi179' 2024-05-25T05:57:37.484 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:37 smithi179 ceph-mon[25623]: from='osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T05:57:37.484 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:37 smithi179 ceph-mon[25623]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T05:57:37.484 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:37.484 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:37.484 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:37.484 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:37.506 DEBUG:teuthology.orchestra.run.smithi179:osd.5> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.5.service 2024-05-25T05:57:37.508 INFO:tasks.cephadm:Deploying osd.6 on smithi179 with /dev/vg_nvme/lv_2... 2024-05-25T05:57:37.509 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-25T05:57:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[24776]: from='osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T05:57:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[24776]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T05:57:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[29436]: from='osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T05:57:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[29436]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T05:57:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:38.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:38 smithi179 ceph-mon[25623]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:38.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:38 smithi179 ceph-mon[25623]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-25T05:57:38.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:38 smithi179 ceph-mon[25623]: osdmap e37: 6 total, 5 up, 6 in 2024-05-25T05:57:38.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:38 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:38.725 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:38 smithi179 ceph-mon[25623]: from='osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:38.725 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:38 smithi179 ceph-mon[25623]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:38.725 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 05:57:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[31907]: 2024-05-25T05:57:38.451+0000 7ff124fa4700 -1 osd.5 0 waiting for initial osdmap 2024-05-25T05:57:38.725 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 05:57:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[31907]: 2024-05-25T05:57:38.456+0000 7ff12093d700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:57:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[24776]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[24776]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-25T05:57:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[24776]: osdmap e37: 6 total, 5 up, 6 in 2024-05-25T05:57:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[24776]: from='osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[24776]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[29436]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[29436]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-25T05:57:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[29436]: osdmap e37: 6 total, 5 up, 6 in 2024-05-25T05:57:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[29436]: from='osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:38 smithi175 ceph-mon[29436]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: osdmap e38: 6 total, 5 up, 6 in 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263] boot 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: osdmap e39: 6 total, 6 up, 6 in 2024-05-25T05:57:39.601 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:39 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: osdmap e38: 6 total, 5 up, 6 in 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263] boot 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: osdmap e39: 6 total, 6 up, 6 in 2024-05-25T05:57:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: osdmap e38: 6 total, 5 up, 6 in 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: osd.5 [v2:172.21.15.179:6808/1348319263,v1:172.21.15.179:6809/1348319263] boot 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: osdmap e39: 6 total, 6 up, 6 in 2024-05-25T05:57:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:39 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:57:40.714 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:40 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:57:40.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:40 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:57:40.715 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:40 smithi179 ceph-mon[25623]: osdmap e40: 6 total, 6 up, 6 in 2024-05-25T05:57:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:40 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:57:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:40 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:57:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:40 smithi175 ceph-mon[29436]: osdmap e40: 6 total, 6 up, 6 in 2024-05-25T05:57:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:40 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:57:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:40 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:57:40.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:40 smithi175 ceph-mon[24776]: osdmap e40: 6 total, 6 up, 6 in 2024-05-25T05:57:41.459 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-25T05:57:41.459 INFO:teuthology.orchestra.run.smithi179.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-25T05:57:41.459 INFO:teuthology.orchestra.run.smithi179.stderr: stderr: 10+0 records in 2024-05-25T05:57:41.459 INFO:teuthology.orchestra.run.smithi179.stderr:10+0 records out 2024-05-25T05:57:41.459 INFO:teuthology.orchestra.run.smithi179.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0143067 s, 733 MB/s 2024-05-25T05:57:41.460 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping successful for: 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi179 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi179 to 8544M 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:41 smithi179 ceph-mon[25623]: osdmap e41: 6 total, 6 up, 6 in 2024-05-25T05:57:42.119 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_2 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi179 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi179 to 8544M 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[24776]: osdmap e41: 6 total, 6 up, 6 in 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi179 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi179 to 8544M 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:42.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:41 smithi175 ceph-mon[29436]: osdmap e41: 6 total, 6 up, 6 in 2024-05-25T05:57:43.646 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:43 smithi179 ceph-mon[25623]: osdmap e42: 6 total, 6 up, 6 in 2024-05-25T05:57:43.646 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:43 smithi179 ceph-mon[25623]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:43 smithi175 ceph-mon[24776]: osdmap e42: 6 total, 6 up, 6 in 2024-05-25T05:57:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:43 smithi175 ceph-mon[24776]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:43 smithi175 ceph-mon[29436]: osdmap e42: 6 total, 6 up, 6 in 2024-05-25T05:57:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:43 smithi175 ceph-mon[29436]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:44.698 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:44 smithi179 ceph-mon[25623]: osdmap e43: 6 total, 6 up, 6 in 2024-05-25T05:57:44.698 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:44 smithi179 ceph-mon[25623]: from='client.24254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:57:44.698 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:57:44.698 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:57:44.698 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:44 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[29436]: osdmap e43: 6 total, 6 up, 6 in 2024-05-25T05:57:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[29436]: from='client.24254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:57:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:57:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:57:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[24776]: osdmap e43: 6 total, 6 up, 6 in 2024-05-25T05:57:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[24776]: from='client.24254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:57:44.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:57:44.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:57:44.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:44 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:45.629 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:45 smithi179 ceph-mon[25623]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:45 smithi175 ceph-mon[24776]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:45 smithi175 ceph-mon[29436]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:46 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/4018763557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]: dispatch 2024-05-25T05:57:46.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:46 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]: dispatch 2024-05-25T05:57:46.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:46 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]': finished 2024-05-25T05:57:46.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:46 smithi179 ceph-mon[25623]: osdmap e44: 7 total, 6 up, 7 in 2024-05-25T05:57:46.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:46 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:57:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/4018763557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]: dispatch 2024-05-25T05:57:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]: dispatch 2024-05-25T05:57:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]': finished 2024-05-25T05:57:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[24776]: osdmap e44: 7 total, 6 up, 7 in 2024-05-25T05:57:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:57:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/4018763557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]: dispatch 2024-05-25T05:57:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]: dispatch 2024-05-25T05:57:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a6bcc838-0f33-48b1-ab5c-765e38d071f2"}]': finished 2024-05-25T05:57:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[29436]: osdmap e44: 7 total, 6 up, 7 in 2024-05-25T05:57:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:46 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:57:47.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:47 smithi179 ceph-mon[25623]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:47.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:47 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/135764502' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:47 smithi175 ceph-mon[24776]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:47 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/135764502' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:47 smithi175 ceph-mon[29436]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:47.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:47 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/135764502' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:57:50.456 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:50 smithi179 ceph-mon[25623]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:50 smithi175 ceph-mon[24776]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:50.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:50 smithi175 ceph-mon[29436]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:52.492 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:52 smithi179 ceph-mon[25623]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:52 smithi175 ceph-mon[24776]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:52 smithi175 ceph-mon[29436]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:53.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:53 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T05:57:53.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:53 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:53 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T05:57:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:53 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:53 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T05:57:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:53 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:54 smithi179 ceph-mon[25623]: Deploying daemon osd.6 on smithi179 2024-05-25T05:57:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:54 smithi179 ceph-mon[25623]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:54 smithi175 ceph-mon[24776]: Deploying daemon osd.6 on smithi179 2024-05-25T05:57:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:54 smithi175 ceph-mon[24776]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:54 smithi175 ceph-mon[29436]: Deploying daemon osd.6 on smithi179 2024-05-25T05:57:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:54 smithi175 ceph-mon[29436]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:56.696 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:56 smithi179 ceph-mon[25623]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:56 smithi175 ceph-mon[29436]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:56 smithi175 ceph-mon[24776]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:58.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:58 smithi179 ceph-mon[25623]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:58.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:58 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:58.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:58 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:58.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:58 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:58.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:57:58 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[24776]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:57:58.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[29436]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:57:58.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:57:58.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:57:58.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:57:58.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:57:58 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:00.582 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:00 smithi179 ceph-mon[25623]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:00.616 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 6 on host 'smithi179' 2024-05-25T05:58:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:00 smithi175 ceph-mon[24776]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:00 smithi175 ceph-mon[29436]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:01.213 DEBUG:teuthology.orchestra.run.smithi179:osd.6> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.6.service 2024-05-25T05:58:01.216 INFO:tasks.cephadm:Deploying osd.7 on smithi179 with /dev/vg_nvme/lv_1... 2024-05-25T05:58:01.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-25T05:58:01.742 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:01 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:01.743 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:01 smithi179 ceph-mon[25623]: from='osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T05:58:01.743 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:01 smithi179 ceph-mon[25623]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:01.743 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:01 smithi179 ceph-mon[25623]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T05:58:01.743 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:01 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[24776]: from='osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T05:58:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[24776]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[24776]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T05:58:02.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:02.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:02.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[29436]: from='osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T05:58:02.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[29436]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:02.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[29436]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T05:58:02.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:01 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:02 smithi179 ceph-mon[25623]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-25T05:58:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:02 smithi179 ceph-mon[25623]: osdmap e45: 7 total, 6 up, 7 in 2024-05-25T05:58:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:02 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:03.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:02 smithi179 ceph-mon[25623]: from='osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:03.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:02 smithi179 ceph-mon[25623]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:03.036 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 05:58:02 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[34784]: 2024-05-25T05:58:02.627+0000 7f9504b60700 -1 osd.6 0 waiting for initial osdmap 2024-05-25T05:58:03.036 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 05:58:02 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[34784]: 2024-05-25T05:58:02.635+0000 7f94fecf6700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:58:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[24776]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-25T05:58:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[24776]: osdmap e45: 7 total, 6 up, 7 in 2024-05-25T05:58:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[24776]: from='osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[24776]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:03.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[29436]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-25T05:58:03.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[29436]: osdmap e45: 7 total, 6 up, 7 in 2024-05-25T05:58:03.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:03.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[29436]: from='osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:03.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:02 smithi175 ceph-mon[29436]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:03.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:03 smithi179 ceph-mon[25623]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:58:03.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:03 smithi179 ceph-mon[25623]: osdmap e46: 7 total, 6 up, 7 in 2024-05-25T05:58:03.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:03.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:03 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:03.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:03 smithi179 ceph-mon[25623]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[29436]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:58:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[29436]: osdmap e46: 7 total, 6 up, 7 in 2024-05-25T05:58:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[29436]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:04.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[24776]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:58:04.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[24776]: osdmap e46: 7 total, 6 up, 7 in 2024-05-25T05:58:04.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:04.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:04.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:03 smithi175 ceph-mon[24776]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-25T05:58:04.800 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-25T05:58:04.800 INFO:teuthology.orchestra.run.smithi179.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-25T05:58:04.800 INFO:teuthology.orchestra.run.smithi179.stderr: stderr: 10+0 records in 2024-05-25T05:58:04.800 INFO:teuthology.orchestra.run.smithi179.stderr:10+0 records out 2024-05-25T05:58:04.800 INFO:teuthology.orchestra.run.smithi179.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012413 s, 845 MB/s 2024-05-25T05:58:04.800 INFO:teuthology.orchestra.run.smithi179.stderr:--> Zapping successful for: 2024-05-25T05:58:05.009 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:04 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:58:05.010 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:04 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:58:05.010 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:04 smithi179 ceph-mon[25623]: osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289] boot 2024-05-25T05:58:05.010 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:04 smithi179 ceph-mon[25623]: osdmap e47: 7 total, 7 up, 7 in 2024-05-25T05:58:05.010 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:04 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:05.010 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:04 smithi179 ceph-mon[25623]: osdmap e48: 7 total, 7 up, 7 in 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[24776]: osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289] boot 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[24776]: osdmap e47: 7 total, 7 up, 7 in 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[24776]: osdmap e48: 7 total, 7 up, 7 in 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:58:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:58:05.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[29436]: osd.6 [v2:172.21.15.179:6816/1139227289,v1:172.21.15.179:6817/1139227289] boot 2024-05-25T05:58:05.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[29436]: osdmap e47: 7 total, 7 up, 7 in 2024-05-25T05:58:05.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:05.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:04 smithi175 ceph-mon[29436]: osdmap e48: 7 total, 7 up, 7 in 2024-05-25T05:58:05.484 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_1 2024-05-25T05:58:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: osdmap e49: 7 total, 7 up, 7 in 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:06 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: osdmap e49: 7 total, 7 up, 7 in 2024-05-25T05:58:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: osdmap e49: 7 total, 7 up, 7 in 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:06 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:07.506 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:07 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi179 2024-05-25T05:58:07.507 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:07 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi179 to 5696M 2024-05-25T05:58:07.507 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:07 smithi179 ceph-mon[25623]: osdmap e50: 7 total, 7 up, 7 in 2024-05-25T05:58:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:07 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi179 2024-05-25T05:58:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:07 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi179 to 5696M 2024-05-25T05:58:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:07 smithi175 ceph-mon[24776]: osdmap e50: 7 total, 7 up, 7 in 2024-05-25T05:58:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:07 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi179 2024-05-25T05:58:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:07 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi179 to 5696M 2024-05-25T05:58:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:07 smithi175 ceph-mon[29436]: osdmap e50: 7 total, 7 up, 7 in 2024-05-25T05:58:08.618 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:08 smithi179 ceph-mon[25623]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:08.618 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:08 smithi179 ceph-mon[25623]: osdmap e51: 7 total, 7 up, 7 in 2024-05-25T05:58:08.618 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:58:08.618 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:58:08.618 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:08 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[24776]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[24776]: osdmap e51: 7 total, 7 up, 7 in 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[29436]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[29436]: osdmap e51: 7 total, 7 up, 7 in 2024-05-25T05:58:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-25T05:58:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-25T05:58:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:08 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:09.499 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:09 smithi179 ceph-mon[25623]: from='client.24281 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:09.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:09 smithi175 ceph-mon[24776]: from='client.24281 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:09 smithi175 ceph-mon[29436]: from='client.24281 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:10 smithi179 ceph-mon[25623]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:10 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/4230114863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]: dispatch 2024-05-25T05:58:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:10 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]: dispatch 2024-05-25T05:58:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:10 smithi179 ceph-mon[25623]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]': finished 2024-05-25T05:58:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:10 smithi179 ceph-mon[25623]: osdmap e52: 8 total, 7 up, 8 in 2024-05-25T05:58:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:10 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[29436]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/4230114863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]: dispatch 2024-05-25T05:58:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]: dispatch 2024-05-25T05:58:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[29436]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]': finished 2024-05-25T05:58:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[29436]: osdmap e52: 8 total, 7 up, 8 in 2024-05-25T05:58:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[24776]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:10.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/4230114863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]: dispatch 2024-05-25T05:58:10.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]: dispatch 2024-05-25T05:58:10.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[24776]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa0ad12f-3ba1-4e91-acfb-a09beac21098"}]': finished 2024-05-25T05:58:10.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[24776]: osdmap e52: 8 total, 7 up, 8 in 2024-05-25T05:58:10.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:10 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:11 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/3038354124' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:58:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:11 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/3038354124' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:58:11.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:11 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/3038354124' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-25T05:58:12.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:12 smithi179 ceph-mon[25623]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-05-25T05:58:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:12 smithi175 ceph-mon[29436]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-05-25T05:58:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:12 smithi175 ceph-mon[24776]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-05-25T05:58:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:14 smithi179 ceph-mon[25623]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-25T05:58:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:14 smithi175 ceph-mon[24776]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-25T05:58:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:14 smithi175 ceph-mon[29436]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-25T05:58:16.685 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:16 smithi179 ceph-mon[25623]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-25T05:58:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:16 smithi175 ceph-mon[24776]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-25T05:58:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:16 smithi175 ceph-mon[29436]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-25T05:58:17.638 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:17 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T05:58:17.638 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:17 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:17.638 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:17 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T05:58:17.638 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:17 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:17.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:17 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T05:58:17.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:17 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:18.672 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:18 smithi179 ceph-mon[25623]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-05-25T05:58:18.673 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:18 smithi179 ceph-mon[25623]: Deploying daemon osd.7 on smithi179 2024-05-25T05:58:18.784 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:18 smithi175 ceph-mon[29436]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-05-25T05:58:18.785 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:18 smithi175 ceph-mon[29436]: Deploying daemon osd.7 on smithi179 2024-05-25T05:58:18.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:18 smithi175 ceph-mon[24776]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-05-25T05:58:18.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:18 smithi175 ceph-mon[24776]: Deploying daemon osd.7 on smithi179 2024-05-25T05:58:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:20 smithi179 ceph-mon[25623]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-25T05:58:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:20 smithi175 ceph-mon[29436]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-25T05:58:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:20 smithi175 ceph-mon[24776]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-25T05:58:22.496 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:22 smithi179 ceph-mon[25623]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:22.496 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:22.496 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:22.497 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:22.497 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:22 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[24776]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:22.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[29436]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:22.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:22.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:22.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:22.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:22 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:23 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:23 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:23 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:24.420 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:24 smithi179 ceph-mon[25623]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:24 smithi175 ceph-mon[24776]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:24.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:24 smithi175 ceph-mon[29436]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:25.334 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 7 on host 'smithi179' 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:25.648 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:25 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T05:58:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T05:58:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:25.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:25 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:25.994 DEBUG:teuthology.orchestra.run.smithi179:osd.7> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.7.service 2024-05-25T05:58:25.997 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-05-25T05:58:25.997 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd stat -f json 2024-05-25T05:58:26.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:26 smithi179 ceph-mon[25623]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:26.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:26 smithi179 ceph-mon[25623]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-25T05:58:26.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:26 smithi179 ceph-mon[25623]: osdmap e53: 8 total, 7 up, 8 in 2024-05-25T05:58:26.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:26 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:26.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:26 smithi179 ceph-mon[25623]: from='osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:26.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:26 smithi179 ceph-mon[25623]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:26.562 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[24776]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:26.562 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[24776]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-25T05:58:26.562 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[24776]: osdmap e53: 8 total, 7 up, 8 in 2024-05-25T05:58:26.562 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:26.562 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[24776]: from='osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:26.563 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[24776]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:26.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[29436]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:26.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[29436]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-25T05:58:26.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[29436]: osdmap e53: 8 total, 7 up, 8 in 2024-05-25T05:58:26.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:26.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[29436]: from='osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:26.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:26 smithi175 ceph-mon[29436]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T05:58:27.135 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 05:58:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[37685]: 2024-05-25T05:58:26.875+0000 7f04bad9f700 -1 osd.7 0 waiting for initial osdmap 2024-05-25T05:58:27.136 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 05:58:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[37685]: 2024-05-25T05:58:26.885+0000 7f04b5f37700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T05:58:27.956 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:58:28.176 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:27 smithi179 ceph-mon[25623]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:28.176 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:27 smithi179 ceph-mon[25623]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:58:28.176 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:27 smithi179 ceph-mon[25623]: osdmap e54: 8 total, 7 up, 8 in 2024-05-25T05:58:28.176 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:27 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:28.176 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:27 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:28.276 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[29436]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:28.276 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[29436]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:58:28.276 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[29436]: osdmap e54: 8 total, 7 up, 8 in 2024-05-25T05:58:28.276 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:28.276 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:28.277 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[24776]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-25T05:58:28.277 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[24776]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-05-25T05:58:28.277 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[24776]: osdmap e54: 8 total, 7 up, 8 in 2024-05-25T05:58:28.277 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:28.277 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:27 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:28.763 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1716616707,"num_in_osds":8,"osd_in_since":1716616690,"num_remapped_pgs":1} 2024-05-25T05:58:28.763 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-05-25T05:58:28.988 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:58:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: purged_snaps scrub starts 2024-05-25T05:58:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: purged_snaps scrub ok 2024-05-25T05:58:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204] boot 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: osdmap e55: 8 total, 8 up, 8 in 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3054435353' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:28 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: purged_snaps scrub starts 2024-05-25T05:58:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: purged_snaps scrub ok 2024-05-25T05:58:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204] boot 2024-05-25T05:58:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: osdmap e55: 8 total, 8 up, 8 in 2024-05-25T05:58:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3054435353' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-25T05:58:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: purged_snaps scrub starts 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: purged_snaps scrub ok 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: osd.7 [v2:172.21.15.179:6824/3524897204,v1:172.21.15.179:6825/3524897204] boot 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: osdmap e55: 8 total, 8 up, 8 in 2024-05-25T05:58:29.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3054435353' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:29.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:28 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[24776]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi179 2024-05-25T05:58:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi179 to 4272M 2024-05-25T05:58:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[24776]: osdmap e56: 8 total, 8 up, 8 in 2024-05-25T05:58:30.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[29436]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:30.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi179 2024-05-25T05:58:30.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi179 to 4272M 2024-05-25T05:58:30.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:29 smithi175 ceph-mon[29436]: osdmap e56: 8 total, 8 up, 8 in 2024-05-25T05:58:30.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:29 smithi179 ceph-mon[25623]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:30.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:29 smithi179 ceph-mon[25623]: Detected new or changed devices on smithi179 2024-05-25T05:58:30.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:29 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi179 to 4272M 2024-05-25T05:58:30.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:29 smithi179 ceph-mon[25623]: osdmap e56: 8 total, 8 up, 8 in 2024-05-25T05:58:30.808 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:58:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":57,"fsid":"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968","created":"2024-05-25T05:53:35.188638+0000","modified":"2024-05-25T05:58:29.889808+0000","last_up_change":"2024-05-25T05:58:27.869955+0000","last_in_change":"2024-05-25T05:58:10.214659+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-25T05:56:32.911195+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"127d9a00-0ee0-4842-bf83-11a1f4a93fe8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6803","nonce":2524900476}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6805","nonce":2524900476}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6809","nonce":2524900476}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6807","nonce":2524900476}]},"public_addr":"172.21.15.175:6803/2524900476","cluster_addr":"172.21.15.175:6805/2524900476","heartbeat_back_addr":"172.21.15.175:6809/2524900476","heartbeat_front_addr":"172.21.15.175:6807/2524900476","state":["exists","up"]},{"osd":1,"uuid":"1a2ed55d-9db0-4df8-be29-d0a3313205ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6811","nonce":417501965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6813","nonce":417501965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6817","nonce":417501965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6815","nonce":417501965}]},"public_addr":"172.21.15.175:6811/417501965","cluster_addr":"172.21.15.175:6813/417501965","heartbeat_back_addr":"172.21.15.175:6817/417501965","heartbeat_front_addr":"172.21.15.175:6815/417501965","state":["exists","up"]},{"osd":2,"uuid":"3fad27f8-b98f-43f6-a52e-f64ed125afd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6819","nonce":4111967520}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6821","nonce":4111967520}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6825","nonce":4111967520}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6823","nonce":4111967520}]},"public_addr":"172.21.15.175:6819/4111967520","cluster_addr":"172.21.15.175:6821/4111967520","heartbeat_back_addr":"172.21.15.175:6825/4111967520","heartbeat_front_addr":"172.21.15.175:6823/4111967520","state":["exists","up"]},{"osd":3,"uuid":"48140d40-4271-49e8-9e16-afb39b0de88f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6827","nonce":1737862094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6829","nonce":1737862094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6832","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6833","nonce":1737862094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6831","nonce":1737862094}]},"public_addr":"172.21.15.175:6827/1737862094","cluster_addr":"172.21.15.175:6829/1737862094","heartbeat_back_addr":"172.21.15.175:6833/1737862094","heartbeat_front_addr":"172.21.15.175:6831/1737862094","state":["exists","up"]},{"osd":4,"uuid":"a65572db-45a9-47ee-a565-079c774a48f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6801","nonce":4035633389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6803","nonce":4035633389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6807","nonce":4035633389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6805","nonce":4035633389}]},"public_addr":"172.21.15.179:6801/4035633389","cluster_addr":"172.21.15.179:6803/4035633389","heartbeat_back_addr":"172.21.15.179:6807/4035633389","heartbeat_front_addr":"172.21.15.179:6805/4035633389","state":["exists","up"]},{"osd":5,"uuid":"b8b21121-8638-48a5-884e-93be0d0d8c17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6809","nonce":1348319263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6811","nonce":1348319263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6815","nonce":1348319263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6813","nonce":1348319263}]},"public_addr":"172.21.15.179:6809/1348319263","cluster_addr":"172.21.15.179:6811/1348319263","heartbeat_back_addr":"172.21.15.179:6815/1348319263","heartbeat_front_addr":"172.21.15.179:6813/1348319263","state":["exists","up"]},{"osd":6,"uuid":"a6bcc838-0f33-48b1-ab5c-765e38d071f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6817","nonce":1139227289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6819","nonce":1139227289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6823","nonce":1139227289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6821","nonce":1139227289}]},"public_addr":"172.21.15.179:6817/1139227289","cluster_addr":"172.21.15.179:6819/1139227289","heartbeat_back_addr":"172.21.15.179:6823/1139227289","heartbeat_front_addr":"172.21.15.179:6821/1139227289","state":["exists","up"]},{"osd":7,"uuid":"aa0ad12f-3ba1-4e91-acfb-a09beac21098","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6825","nonce":3524897204}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6827","nonce":3524897204}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6831","nonce":3524897204}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6829","nonce":3524897204}]},"public_addr":"172.21.15.179:6825/3524897204","cluster_addr":"172.21.15.179:6827/3524897204","heartbeat_back_addr":"172.21.15.179:6831/3524897204","heartbeat_front_addr":"172.21.15.179:6829/3524897204","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:55:41.158623+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:05.317786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:28.805740+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:52.651914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:57:14.065233+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:57:37.674617+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:58:01.634012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:58:25.955140+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.175:6801/2269993462":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/2149746729":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/896188687":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/1040639744":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/3928322074":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6800/2269993462":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/393692458":"2024-05-26T05:53:59.401659+0000","172.21.15.175:0/613585740":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6800/1469117619":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6801/1469117619":"2024-05-26T05:53:59.401659+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-25T05:58:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:30 smithi175 ceph-mon[24776]: osdmap e57: 8 total, 8 up, 8 in 2024-05-25T05:58:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:30 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2544954976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:58:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:30 smithi175 ceph-mon[29436]: osdmap e57: 8 total, 8 up, 8 in 2024-05-25T05:58:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:30 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2544954976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:58:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:30 smithi179 ceph-mon[25623]: osdmap e57: 8 total, 8 up, 8 in 2024-05-25T05:58:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:30 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2544954976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:58:31.511 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-25T05:56:32.911195+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-05-25T05:58:31.512 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-25T05:58:31.724 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:58:32.156 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:31 smithi175 ceph-mon[29436]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:32.156 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:31 smithi175 ceph-mon[29436]: osdmap e58: 8 total, 8 up, 8 in 2024-05-25T05:58:32.156 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:31 smithi175 ceph-mon[24776]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:32.157 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:31 smithi175 ceph-mon[24776]: osdmap e58: 8 total, 8 up, 8 in 2024-05-25T05:58:32.186 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:31 smithi179 ceph-mon[25623]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:32.186 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:31 smithi179 ceph-mon[25623]: osdmap e58: 8 total, 8 up, 8 in 2024-05-25T05:58:33.162 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:32 smithi175 ceph-mon[24776]: osdmap e59: 8 total, 8 up, 8 in 2024-05-25T05:58:33.162 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:32 smithi175 ceph-mon[29436]: osdmap e59: 8 total, 8 up, 8 in 2024-05-25T05:58:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:32 smithi179 ceph-mon[25623]: osdmap e59: 8 total, 8 up, 8 in 2024-05-25T05:58:33.543 INFO:teuthology.orchestra.run.smithi175.stdout:pg_num: 1 2024-05-25T05:58:34.218 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:33 smithi175 ceph-mon[29436]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:34.218 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:33 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/431094510' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-25T05:58:34.218 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:33 smithi175 ceph-mon[24776]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:34.218 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:33 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/431094510' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-25T05:58:34.247 INFO:tasks.cephadm:Adding prometheus.a on smithi179 2024-05-25T05:58:34.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch apply prometheus '1;smithi179=a' 2024-05-25T05:58:34.277 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:33 smithi179 ceph-mon[25623]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:34.277 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:33 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/431094510' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-25T05:58:36.022 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled prometheus update... 2024-05-25T05:58:36.581 DEBUG:teuthology.orchestra.run.smithi179:prometheus.a> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service 2024-05-25T05:58:36.624 INFO:tasks.cephadm:Adding node-exporter.a on smithi175 2024-05-25T05:58:36.624 INFO:tasks.cephadm:Adding node-exporter.b on smithi179 2024-05-25T05:58:36.624 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch apply node-exporter '2;smithi175=a;smithi179=b' 2024-05-25T05:58:36.644 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:36 smithi179 ceph-mon[25623]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-25T05:58:36.644 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:36.644 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:36.644 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:36.644 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:36.644 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:36.644 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:36 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-25T05:58:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[24776]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[29436]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:36.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:36.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' 2024-05-25T05:58:36.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:36 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-25T05:58:37.205 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ignoring --setuser ceph since I am not root 2024-05-25T05:58:37.206 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ignoring --setgroup ceph since I am not root 2024-05-25T05:58:37.206 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-mgr[26799]: -- 172.21.15.179:0/3221965637 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55978c7a6340 con 0x55978c77f000 2024-05-25T05:58:37.206 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:37.206+0000 7f360b99e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T05:58:37.340 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setuser ceph since I am not root 2024-05-25T05:58:37.340 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setgroup ceph since I am not root 2024-05-25T05:58:37.340 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mgr[25008]: -- 172.21.15.175:0/3976564871 <== mon.0 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b0f18f6340 con 0x55b0f18cec00 2024-05-25T05:58:37.340 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:37.212+0000 7f056979c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T05:58:37.340 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:37.339+0000 7f056979c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:58:37.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:37 smithi179 ceph-mon[25623]: from='client.24317 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:37.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:37 smithi179 ceph-mon[25623]: Saving service prometheus spec with placement smithi179=a;count:1 2024-05-25T05:58:37.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:37 smithi179 ceph-mon[25623]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-25T05:58:37.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:37 smithi179 ceph-mon[25623]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-25T05:58:37.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:37.340+0000 7f360b99e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:58:37.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:37.438+0000 7f360b99e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T05:58:37.745 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[24776]: from='client.24317 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:37.745 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[24776]: Saving service prometheus spec with placement smithi179=a;count:1 2024-05-25T05:58:37.745 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[24776]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-25T05:58:37.746 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[24776]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-25T05:58:37.746 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:37.440+0000 7f056979c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T05:58:37.746 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[29436]: from='client.24317 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:37.746 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[29436]: Saving service prometheus spec with placement smithi179=a;count:1 2024-05-25T05:58:37.746 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[29436]: from='mgr.14148 172.21.15.175:0/35341537' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-25T05:58:37.746 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:37 smithi175 ceph-mon[29436]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-25T05:58:37.990 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:37.747+0000 7f360b99e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T05:58:37.990 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:37.830+0000 7f360b99e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T05:58:38.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:37.744+0000 7f056979c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T05:58:38.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:37.827+0000 7f056979c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T05:58:39.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:38.978+0000 7f360b99e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T05:58:39.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:39.142+0000 7f360b99e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T05:58:39.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:39.224+0000 7f360b99e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T05:58:39.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:38.937+0000 7f056979c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T05:58:39.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:39.100+0000 7f056979c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T05:58:39.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:39.181+0000 7f056979c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T05:58:39.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:39.496+0000 7f056979c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T05:58:39.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:39.586+0000 7f056979c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T05:58:39.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:39.678+0000 7f056979c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T05:58:39.963 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:39.537+0000 7f360b99e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T05:58:39.963 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:39.627+0000 7f360b99e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T05:58:39.963 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:39.719+0000 7f360b99e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T05:58:39.963 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:39.962+0000 7f360b99e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:58:40.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:39.919+0000 7f056979c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T05:58:40.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:40.398+0000 7f360b99e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T05:58:40.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:40.354+0000 7f056979c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T05:58:41.228 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:40.843+0000 7f056979c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T05:58:41.228 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:40.937+0000 7f056979c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T05:58:41.265 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:40.881+0000 7f360b99e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T05:58:41.265 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:40.973+0000 7f360b99e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T05:58:41.265 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:41.265+0000 7f360b99e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T05:58:41.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:41.355+0000 7f360b99e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T05:58:41.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:41.227+0000 7f056979c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T05:58:41.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:41.317+0000 7f056979c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T05:58:42.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:41.762+0000 7f360b99e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T05:58:42.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:41.893+0000 7f360b99e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T05:58:42.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:41.986+0000 7f360b99e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T05:58:42.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:41.730+0000 7f056979c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T05:58:42.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:41.862+0000 7f056979c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T05:58:42.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:41.956+0000 7f056979c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T05:58:42.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:42.069+0000 7f056979c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T05:58:42.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:42.158+0000 7f056979c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T05:58:42.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:42 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:42.101+0000 7f360b99e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T05:58:42.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:42 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:42.189+0000 7f360b99e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T05:58:43.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:42 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:42.741+0000 7f360b99e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T05:58:43.067 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:42.706+0000 7f056979c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T05:58:43.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:43.067+0000 7f056979c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T05:58:43.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:58:43] ENGINE Bus STARTING 2024-05-25T05:58:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: Active manager daemon y restarted 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: Activating manager daemon y 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: osdmap e60: 8 total, 8 up, 8 in 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: Standby manager daemon x restarted 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T05:58:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: Active manager daemon y restarted 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: Activating manager daemon y 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: osdmap e60: 8 total, 8 up, 8 in 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: Standby manager daemon x restarted 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: Standby manager daemon x started 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T05:58:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:43 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: Active manager daemon y restarted 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: Activating manager daemon y 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: osdmap e60: 8 total, 8 up, 8 in 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: Standby manager daemon x restarted 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: Standby manager daemon x started 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T05:58:43.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:43 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/287242394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T05:58:43.441 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:43.111+0000 7f360b99e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T05:58:43.441 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: [25/May/2024:05:58:43] ENGINE Bus STARTING 2024-05-25T05:58:43.441 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: CherryPy Checker: 2024-05-25T05:58:43.441 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: The Application mounted at '' has an empty config. 2024-05-25T05:58:43.441 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: 2024-05-25T05:58:43.441 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: [25/May/2024:05:58:43] ENGINE Serving on http://:::9283 2024-05-25T05:58:43.441 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:58:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: [25/May/2024:05:58:43] ENGINE Bus STARTED 2024-05-25T05:58:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: CherryPy Checker: 2024-05-25T05:58:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: The Application mounted at '' has an empty config. 2024-05-25T05:58:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T05:58:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:58:43] ENGINE Serving on http://:::9283 2024-05-25T05:58:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:58:43] ENGINE Bus STARTED 2024-05-25T05:58:44.150 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled node-exporter update... 2024-05-25T05:58:44.208 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:58:44] ENGINE Bus STARTING 2024-05-25T05:58:44.208 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:58:44] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:58:44.208 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: mgrmap e17: y(active, starting, since 0.0549309s), standbys: x 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T05:58:44.209 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: Manager daemon y is now available 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:44.210 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[29436]: [25/May/2024:05:58:44] ENGINE Bus STARTING 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: mgrmap e17: y(active, starting, since 0.0549309s), standbys: x 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:44.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: Manager daemon y is now available 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:44.212 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:44 smithi175 ceph-mon[24776]: [25/May/2024:05:58:44] ENGINE Bus STARTING 2024-05-25T05:58:44.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: mgrmap e17: y(active, starting, since 0.0549309s), standbys: x 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T05:58:44.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: Manager daemon y is now available 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:58:44.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:44 smithi179 ceph-mon[25623]: [25/May/2024:05:58:44] ENGINE Bus STARTING 2024-05-25T05:58:44.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:58:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:05:58:44] ENGINE Bus STARTED 2024-05-25T05:58:44.907 DEBUG:teuthology.orchestra.run.smithi175:node-exporter.a> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service 2024-05-25T05:58:44.909 DEBUG:teuthology.orchestra.run.smithi179:node-exporter.b> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service 2024-05-25T05:58:44.912 INFO:tasks.cephadm:Adding alertmanager.a on smithi175 2024-05-25T05:58:44.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch apply alertmanager '1;smithi175=a' 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: mgrmap e18: y(active, since 1.06852s), standbys: x 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi175=a;smithi179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: Saving service node-exporter spec with placement smithi175=a;smithi179=b;count:2 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: [25/May/2024:05:58:44] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: [25/May/2024:05:58:44] ENGINE Bus STARTED 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.386 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: mgrmap e18: y(active, since 1.06852s), standbys: x 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi175=a;smithi179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: Saving service node-exporter spec with placement smithi175=a;smithi179=b;count:2 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: [25/May/2024:05:58:44] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: [25/May/2024:05:58:44] ENGINE Bus STARTED 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.387 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: mgrmap e18: y(active, since 1.06852s), standbys: x 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi175=a;smithi179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: Saving service node-exporter spec with placement smithi175=a;smithi179=b;count:2 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: [25/May/2024:05:58:44] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: [25/May/2024:05:58:44] ENGINE Bus STARTED 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:45.403 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:46.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:46 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T05:58:46.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:46 smithi179 ceph-mon[25623]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:46.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:46 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:46.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:46 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:58:46.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:46 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:46.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:46 smithi179 ceph-mon[25623]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[29436]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[29436]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T05:58:46.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[24776]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:46.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:46.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:58:46.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:46.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:46 smithi175 ceph-mon[24776]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-25T05:58:46.882 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled alertmanager update... 2024-05-25T05:58:47.621 DEBUG:teuthology.orchestra.run.smithi175:alertmanager.a> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service 2024-05-25T05:58:47.623 INFO:tasks.cephadm:Adding grafana.a on smithi179 2024-05-25T05:58:47.624 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph orch apply grafana '1;smithi179=a' 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi175=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: Saving service alertmanager spec with placement smithi175=a;count:1 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:47.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:47 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:58:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi175=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: Saving service alertmanager spec with placement smithi175=a;count:1 2024-05-25T05:58:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi175=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: Saving service alertmanager spec with placement smithi175=a;count:1 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:47 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:48 smithi179 ceph-mon[25623]: Deploying daemon prometheus.a on smithi179 2024-05-25T05:58:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:48 smithi179 ceph-mon[25623]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:49.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:48 smithi175 ceph-mon[24776]: Deploying daemon prometheus.a on smithi179 2024-05-25T05:58:49.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:48 smithi175 ceph-mon[24776]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:48 smithi175 ceph-mon[29436]: Deploying daemon prometheus.a on smithi179 2024-05-25T05:58:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:48 smithi175 ceph-mon[29436]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:50.617 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled grafana update... 2024-05-25T05:58:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:50 smithi179 ceph-mon[25623]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:50 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:50 smithi175 ceph-mon[24776]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:50 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:51.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:50 smithi175 ceph-mon[29436]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:51.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:50 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:58:51.588 DEBUG:teuthology.orchestra.run.smithi179:grafana.a> sudo journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@grafana.a.service 2024-05-25T05:58:51.590 INFO:tasks.cephadm:Setting up client nodes... 2024-05-25T05:58:51.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-25T05:58:51.904 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:51 smithi175 ceph-mon[29436]: from='client.24371 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:51.904 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:51 smithi175 ceph-mon[29436]: Saving service grafana spec with placement smithi179=a;count:1 2024-05-25T05:58:51.904 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:51 smithi175 ceph-mon[24776]: from='client.24371 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:51.904 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:51 smithi175 ceph-mon[24776]: Saving service grafana spec with placement smithi179=a;count:1 2024-05-25T05:58:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:51 smithi179 ceph-mon[25623]: from='client.24371 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T05:58:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:51 smithi179 ceph-mon[25623]: Saving service grafana spec with placement smithi179=a;count:1 2024-05-25T05:58:52.963 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:52 smithi175 ceph-mon[24776]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:52.964 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:52 smithi175 ceph-mon[29436]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:52 smithi179 ceph-mon[25623]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:53.718 INFO:teuthology.orchestra.run.smithi175.stdout:[client.0] 2024-05-25T05:58:53.718 INFO:teuthology.orchestra.run.smithi175.stdout: key = AQAdflFmjXiWKhAAbUSK2Df3FHJ96MHex2sZhw== 2024-05-25T05:58:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:53 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3960225687' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:53 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3960225687' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-25T05:58:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:53 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3960225687' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:53 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3960225687' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-25T05:58:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:53 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3960225687' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:53 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3960225687' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-25T05:58:54.463 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-05-25T05:58:54.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-25T05:58:54.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-25T05:58:54.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-25T05:58:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:54 smithi179 ceph-mon[25623]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:54 smithi175 ceph-mon[24776]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:55.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:54 smithi175 ceph-mon[29436]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:56.910 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:56 smithi179 ceph-mon[25623]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:57.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:56 smithi175 ceph-mon[24776]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:57.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:56 smithi175 ceph-mon[29436]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:58.704 INFO:teuthology.orchestra.run.smithi179.stdout:[client.1] 2024-05-25T05:58:58.704 INFO:teuthology.orchestra.run.smithi179.stdout: key = AQAiflFmOjq4KRAAKTqbJZLtUVjwGUJrFWwczA== 2024-05-25T05:58:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:58 smithi179 ceph-mon[25623]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:58 smithi179 ceph-mon[25623]: from='client.? 172.21.15.179:0/1772765857' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:58 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:58:58 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-25T05:58:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[24776]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[24776]: from='client.? 172.21.15.179:0/1772765857' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-25T05:58:59.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[29436]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:58:59.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[29436]: from='client.? 172.21.15.179:0/1772765857' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:59.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-25T05:58:59.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:58:58 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-25T05:58:59.360 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:58:59 smithi179 systemd[1]: Starting Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T05:58:59.666 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-05-25T05:58:59.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-05-25T05:58:59.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-05-25T05:58:59.703 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-25T05:58:59.704 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-25T05:58:59.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph mgr dump --format=json 2024-05-25T05:58:59.924 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:00.155 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:58:59 smithi179 podman[40654]: 2024-05-25 05:58:59.902743863 +0000 UTC m=+0.109767836 container create b1d4538f4f2b2b004b7e5805ad1f37107da97f1b7fdb8134cc81432106e6da85 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:00.155 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:58:59 smithi179 podman[40654]: 2024-05-25 05:58:59.810931736 +0000 UTC m=+0.017955720 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-25T05:59:00.155 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 podman[40654]: 2024-05-25 05:59:00.155366029 +0000 UTC m=+0.362390006 container init b1d4538f4f2b2b004b7e5805ad1f37107da97f1b7fdb8134cc81432106e6da85 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:00.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 podman[40654]: 2024-05-25 05:59:00.158235068 +0000 UTC m=+0.365259052 container start b1d4538f4f2b2b004b7e5805ad1f37107da97f1b7fdb8134cc81432106e6da85 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:00.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.181Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-25T05:59:00.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.182Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-25T05:59:00.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.182Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-25T05:59:00.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.182Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi179 (none))" 2024-05-25T05:59:00.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.182Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-25T05:59:00.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.182Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-25T05:59:00.538 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.183Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-25T05:59:00.538 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.183Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-25T05:59:00.538 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.184Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-25T05:59:00.538 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.185Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-25T05:59:00.538 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.185Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.115µs 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.185Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.185Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.185Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.121µs wal_replay_duration=118.176µs total_replay_duration=170.896µs 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.186Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.186Z caller=main.go:947 level=info msg="TSDB started" 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.186Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.202Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.041287ms db_storage=631ns remote_storage=1.05µs web_handler=221ns query_engine=619ns scrape=607.948µs scrape_sd=14.883µs notify=831ns notify_sd=1.265µs rules=15.179063ms 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T05:59:00.202Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 bash[40654]: b1d4538f4f2b2b004b7e5805ad1f37107da97f1b7fdb8134cc81432106e6da85 2024-05-25T05:59:00.539 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 05:59:00 smithi179 systemd[1]: Started Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:59:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:00 smithi179 ceph-mon[25623]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:00 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:01.047 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:00 smithi175 ceph-mon[29436]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:01.047 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:00 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:01.047 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:00 smithi175 ceph-mon[24776]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:01.047 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:00 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:01.048 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:01 smithi175 systemd[1]: Starting Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T05:59:01.895 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:59:01.895 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:01 smithi175 ceph-mon[24776]: Deploying daemon node-exporter.a on smithi175 2024-05-25T05:59:01.896 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:01 smithi175 ceph-mon[29436]: Deploying daemon node-exporter.a on smithi175 2024-05-25T05:59:01.897 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:01 smithi175 bash[44085]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-05-25T05:59:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:01 smithi179 ceph-mon[25623]: Deploying daemon node-exporter.a on smithi175 2024-05-25T05:59:02.309 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:02 smithi175 bash[44085]: Getting image source signatures 2024-05-25T05:59:02.309 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:02 smithi175 bash[44085]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-05-25T05:59:02.309 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:02 smithi175 bash[44085]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-05-25T05:59:02.309 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:02 smithi175 bash[44085]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-25T05:59:02.847 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":19,"active_gid":24340,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":1129599620},{"type":"v1","addr":"172.21.15.175:6801","nonce":1129599620}]},"active_addr":"172.21.15.175:6801/1129599620","active_change":"2024-05-25T05:58:43.070375+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24323,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.175:8443/","prometheus":"http://172.21.15.175:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.175:0","nonce":4130354415}]},{"addrvec":[{"type":"v2","addr":"172.21.15.175:0","nonce":994931434}]},{"addrvec":[{"type":"v2","addr":"172.21.15.175:0","nonce":4138469446}]},{"addrvec":[{"type":"v2","addr":"172.21.15.175:0","nonce":3840021495}]}]}} 2024-05-25T05:59:02.852 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-25T05:59:02.852 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-25T05:59:02.852 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-05-25T05:59:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:02 smithi179 ceph-mon[25623]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:02 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2857342067' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T05:59:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:02 smithi175 ceph-mon[29436]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:02 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2857342067' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T05:59:03.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:02 smithi175 ceph-mon[24776]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:03.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:02 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2857342067' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T05:59:03.687 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:04.058 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:03 smithi175 bash[44085]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-25T05:59:04.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-mon[24776]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:04.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:04.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:04 smithi175 ceph-mon[29436]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:04.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:04 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:04.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 bash[44085]: Writing manifest to image destination 2024-05-25T05:59:04.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:04 smithi179 ceph-mon[25623]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:04.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:04 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:04.974 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 podman[44085]: 2024-05-25 05:59:04.606173558 +0000 UTC m=+2.959564798 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-25T05:59:04.974 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 podman[44085]: 2024-05-25 05:59:04.771807418 +0000 UTC m=+3.125198652 container create cbca05d03edbb888a98756eff7a41eb2fa860722666a537e2a47f21e2e6e36e1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:04.974 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 podman[44085]: 2024-05-25 05:59:04.97424417 +0000 UTC m=+3.327635424 container init cbca05d03edbb888a98756eff7a41eb2fa860722666a537e2a47f21e2e6e36e1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:05.228 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 podman[44085]: 2024-05-25 05:59:04.977691484 +0000 UTC m=+3.331082730 container start cbca05d03edbb888a98756eff7a41eb2fa860722666a537e2a47f21e2e6e36e1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:05.228 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.981Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-25T05:59:05.228 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.981Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.981Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.981Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=arp 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-25T05:59:05.229 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=edac 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-25T05:59:05.230 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=os 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=stat 2024-05-25T05:59:05.231 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=time 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=uname 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[44239]: ts=2024-05-25T05:59:04.982Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:05 smithi175 bash[44085]: cbca05d03edbb888a98756eff7a41eb2fa860722666a537e2a47f21e2e6e36e1 2024-05-25T05:59:05.232 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 05:59:05 smithi175 systemd[1]: Started Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:59:05.461 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:59:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T05:59:06.165 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:06 smithi179 bash[40889]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-05-25T05:59:06.260 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:59:06.261 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:06 smithi175 ceph-mon[24776]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:06.261 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:06 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:06.261 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:06 smithi175 ceph-mon[24776]: Deploying daemon node-exporter.b on smithi179 2024-05-25T05:59:06.261 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:06 smithi175 ceph-mon[29436]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:06.261 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:06 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:06.261 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:06 smithi175 ceph-mon[29436]: Deploying daemon node-exporter.b on smithi179 2024-05-25T05:59:06.261 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":60,"fsid":"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968","created":"2024-05-25T05:53:35.188638+0000","modified":"2024-05-25T05:58:43.070052+0000","last_up_change":"2024-05-25T05:58:27.869955+0000","last_in_change":"2024-05-25T05:58:10.214659+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-25T05:56:32.911195+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"127d9a00-0ee0-4842-bf83-11a1f4a93fe8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6803","nonce":2524900476}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6805","nonce":2524900476}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6809","nonce":2524900476}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6807","nonce":2524900476}]},"public_addr":"172.21.15.175:6803/2524900476","cluster_addr":"172.21.15.175:6805/2524900476","heartbeat_back_addr":"172.21.15.175:6809/2524900476","heartbeat_front_addr":"172.21.15.175:6807/2524900476","state":["exists","up"]},{"osd":1,"uuid":"1a2ed55d-9db0-4df8-be29-d0a3313205ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6811","nonce":417501965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6813","nonce":417501965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6817","nonce":417501965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6815","nonce":417501965}]},"public_addr":"172.21.15.175:6811/417501965","cluster_addr":"172.21.15.175:6813/417501965","heartbeat_back_addr":"172.21.15.175:6817/417501965","heartbeat_front_addr":"172.21.15.175:6815/417501965","state":["exists","up"]},{"osd":2,"uuid":"3fad27f8-b98f-43f6-a52e-f64ed125afd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6819","nonce":4111967520}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6821","nonce":4111967520}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6825","nonce":4111967520}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6823","nonce":4111967520}]},"public_addr":"172.21.15.175:6819/4111967520","cluster_addr":"172.21.15.175:6821/4111967520","heartbeat_back_addr":"172.21.15.175:6825/4111967520","heartbeat_front_addr":"172.21.15.175:6823/4111967520","state":["exists","up"]},{"osd":3,"uuid":"48140d40-4271-49e8-9e16-afb39b0de88f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6827","nonce":1737862094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6829","nonce":1737862094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6832","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6833","nonce":1737862094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6831","nonce":1737862094}]},"public_addr":"172.21.15.175:6827/1737862094","cluster_addr":"172.21.15.175:6829/1737862094","heartbeat_back_addr":"172.21.15.175:6833/1737862094","heartbeat_front_addr":"172.21.15.175:6831/1737862094","state":["exists","up"]},{"osd":4,"uuid":"a65572db-45a9-47ee-a565-079c774a48f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6801","nonce":4035633389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6803","nonce":4035633389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6807","nonce":4035633389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6805","nonce":4035633389}]},"public_addr":"172.21.15.179:6801/4035633389","cluster_addr":"172.21.15.179:6803/4035633389","heartbeat_back_addr":"172.21.15.179:6807/4035633389","heartbeat_front_addr":"172.21.15.179:6805/4035633389","state":["exists","up"]},{"osd":5,"uuid":"b8b21121-8638-48a5-884e-93be0d0d8c17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6809","nonce":1348319263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6811","nonce":1348319263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6815","nonce":1348319263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6813","nonce":1348319263}]},"public_addr":"172.21.15.179:6809/1348319263","cluster_addr":"172.21.15.179:6811/1348319263","heartbeat_back_addr":"172.21.15.179:6815/1348319263","heartbeat_front_addr":"172.21.15.179:6813/1348319263","state":["exists","up"]},{"osd":6,"uuid":"a6bcc838-0f33-48b1-ab5c-765e38d071f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6817","nonce":1139227289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6819","nonce":1139227289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6823","nonce":1139227289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6821","nonce":1139227289}]},"public_addr":"172.21.15.179:6817/1139227289","cluster_addr":"172.21.15.179:6819/1139227289","heartbeat_back_addr":"172.21.15.179:6823/1139227289","heartbeat_front_addr":"172.21.15.179:6821/1139227289","state":["exists","up"]},{"osd":7,"uuid":"aa0ad12f-3ba1-4e91-acfb-a09beac21098","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6825","nonce":3524897204}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6827","nonce":3524897204}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6831","nonce":3524897204}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6829","nonce":3524897204}]},"public_addr":"172.21.15.179:6825/3524897204","cluster_addr":"172.21.15.179:6827/3524897204","heartbeat_back_addr":"172.21.15.179:6831/3524897204","heartbeat_front_addr":"172.21.15.179:6829/3524897204","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:55:41.158623+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:05.317786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:28.805740+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:52.651914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:57:14.065233+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:57:37.674617+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:58:01.634012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:58:25.955140+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.175:0/2643919840":"2024-05-26T05:58:43.070005+0000","172.21.15.175:6800/857453375":"2024-05-26T05:58:43.070005+0000","172.21.15.175:6801/2269993462":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/2149746729":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/896188687":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/1757996095":"2024-05-26T05:58:43.070005+0000","172.21.15.175:0/449852730":"2024-05-26T05:58:43.070005+0000","172.21.15.175:0/1040639744":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/3928322074":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6800/2269993462":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/393692458":"2024-05-26T05:53:59.401659+0000","172.21.15.175:0/613585740":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6800/1469117619":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6801/857453375":"2024-05-26T05:58:43.070005+0000","172.21.15.175:0/3607309314":"2024-05-26T05:58:43.070005+0000","172.21.15.175:6801/1469117619":"2024-05-26T05:53:59.401659+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-25T05:59:06.446 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:06 smithi179 ceph-mon[25623]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:06.446 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:06 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:06.446 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:06 smithi179 ceph-mon[25623]: Deploying daemon node-exporter.b on smithi179 2024-05-25T05:59:06.785 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:06 smithi179 bash[40889]: Getting image source signatures 2024-05-25T05:59:06.785 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:06 smithi179 bash[40889]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-05-25T05:59:06.785 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:06 smithi179 bash[40889]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-05-25T05:59:06.785 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:06 smithi179 bash[40889]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-25T05:59:06.910 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-25T05:59:06.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-05-25T05:59:07.125 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:07.426 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:07 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1718465877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:59:07.427 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:07 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1718465877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:59:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:07 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1718465877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:59:08.171 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:07 smithi179 bash[40889]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-25T05:59:08.171 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:07 smithi179 bash[40889]: Writing manifest to image destination 2024-05-25T05:59:08.420 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:08 smithi175 ceph-mon[29436]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:08.420 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:08 smithi175 ceph-mon[24776]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:08.476 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-mon[25623]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:08.476 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 podman[40889]: 2024-05-25 05:59:08.279110678 +0000 UTC m=+2.358742303 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-25T05:59:08.476 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 podman[40889]: 2024-05-25 05:59:08.36089184 +0000 UTC m=+2.440523484 container create 31b85a1d8b12426ada30191ebda46a839ab0b91a17f3fafb0b30f7bdaa99fb42 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T05:59:08.861 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:59:08.861 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":60,"fsid":"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968","created":"2024-05-25T05:53:35.188638+0000","modified":"2024-05-25T05:58:43.070052+0000","last_up_change":"2024-05-25T05:58:27.869955+0000","last_in_change":"2024-05-25T05:58:10.214659+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-25T05:56:32.911195+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"127d9a00-0ee0-4842-bf83-11a1f4a93fe8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6803","nonce":2524900476}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6805","nonce":2524900476}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6809","nonce":2524900476}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":2524900476},{"type":"v1","addr":"172.21.15.175:6807","nonce":2524900476}]},"public_addr":"172.21.15.175:6803/2524900476","cluster_addr":"172.21.15.175:6805/2524900476","heartbeat_back_addr":"172.21.15.175:6809/2524900476","heartbeat_front_addr":"172.21.15.175:6807/2524900476","state":["exists","up"]},{"osd":1,"uuid":"1a2ed55d-9db0-4df8-be29-d0a3313205ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6811","nonce":417501965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6813","nonce":417501965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6817","nonce":417501965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":417501965},{"type":"v1","addr":"172.21.15.175:6815","nonce":417501965}]},"public_addr":"172.21.15.175:6811/417501965","cluster_addr":"172.21.15.175:6813/417501965","heartbeat_back_addr":"172.21.15.175:6817/417501965","heartbeat_front_addr":"172.21.15.175:6815/417501965","state":["exists","up"]},{"osd":2,"uuid":"3fad27f8-b98f-43f6-a52e-f64ed125afd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6819","nonce":4111967520}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6821","nonce":4111967520}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6825","nonce":4111967520}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":4111967520},{"type":"v1","addr":"172.21.15.175:6823","nonce":4111967520}]},"public_addr":"172.21.15.175:6819/4111967520","cluster_addr":"172.21.15.175:6821/4111967520","heartbeat_back_addr":"172.21.15.175:6825/4111967520","heartbeat_front_addr":"172.21.15.175:6823/4111967520","state":["exists","up"]},{"osd":3,"uuid":"48140d40-4271-49e8-9e16-afb39b0de88f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6827","nonce":1737862094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6829","nonce":1737862094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6832","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6833","nonce":1737862094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":1737862094},{"type":"v1","addr":"172.21.15.175:6831","nonce":1737862094}]},"public_addr":"172.21.15.175:6827/1737862094","cluster_addr":"172.21.15.175:6829/1737862094","heartbeat_back_addr":"172.21.15.175:6833/1737862094","heartbeat_front_addr":"172.21.15.175:6831/1737862094","state":["exists","up"]},{"osd":4,"uuid":"a65572db-45a9-47ee-a565-079c774a48f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6801","nonce":4035633389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6803","nonce":4035633389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6807","nonce":4035633389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":4035633389},{"type":"v1","addr":"172.21.15.179:6805","nonce":4035633389}]},"public_addr":"172.21.15.179:6801/4035633389","cluster_addr":"172.21.15.179:6803/4035633389","heartbeat_back_addr":"172.21.15.179:6807/4035633389","heartbeat_front_addr":"172.21.15.179:6805/4035633389","state":["exists","up"]},{"osd":5,"uuid":"b8b21121-8638-48a5-884e-93be0d0d8c17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6809","nonce":1348319263}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6811","nonce":1348319263}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6815","nonce":1348319263}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":1348319263},{"type":"v1","addr":"172.21.15.179:6813","nonce":1348319263}]},"public_addr":"172.21.15.179:6809/1348319263","cluster_addr":"172.21.15.179:6811/1348319263","heartbeat_back_addr":"172.21.15.179:6815/1348319263","heartbeat_front_addr":"172.21.15.179:6813/1348319263","state":["exists","up"]},{"osd":6,"uuid":"a6bcc838-0f33-48b1-ab5c-765e38d071f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6817","nonce":1139227289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6819","nonce":1139227289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6823","nonce":1139227289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":1139227289},{"type":"v1","addr":"172.21.15.179:6821","nonce":1139227289}]},"public_addr":"172.21.15.179:6817/1139227289","cluster_addr":"172.21.15.179:6819/1139227289","heartbeat_back_addr":"172.21.15.179:6823/1139227289","heartbeat_front_addr":"172.21.15.179:6821/1139227289","state":["exists","up"]},{"osd":7,"uuid":"aa0ad12f-3ba1-4e91-acfb-a09beac21098","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6825","nonce":3524897204}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6827","nonce":3524897204}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6831","nonce":3524897204}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":3524897204},{"type":"v1","addr":"172.21.15.179:6829","nonce":3524897204}]},"public_addr":"172.21.15.179:6825/3524897204","cluster_addr":"172.21.15.179:6827/3524897204","heartbeat_back_addr":"172.21.15.179:6831/3524897204","heartbeat_front_addr":"172.21.15.179:6829/3524897204","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:55:41.158623+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:05.317786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:28.805740+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:56:52.651914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:57:14.065233+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:57:37.674617+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:58:01.634012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-25T05:58:25.955140+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.175:0/2643919840":"2024-05-26T05:58:43.070005+0000","172.21.15.175:6800/857453375":"2024-05-26T05:58:43.070005+0000","172.21.15.175:6801/2269993462":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/2149746729":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/896188687":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/1757996095":"2024-05-26T05:58:43.070005+0000","172.21.15.175:0/449852730":"2024-05-26T05:58:43.070005+0000","172.21.15.175:0/1040639744":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/3928322074":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6800/2269993462":"2024-05-26T05:54:22.547125+0000","172.21.15.175:0/393692458":"2024-05-26T05:53:59.401659+0000","172.21.15.175:0/613585740":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6800/1469117619":"2024-05-26T05:53:59.401659+0000","172.21.15.175:6801/857453375":"2024-05-26T05:58:43.070005+0000","172.21.15.175:0/3607309314":"2024-05-26T05:58:43.070005+0000","172.21.15.175:6801/1469117619":"2024-05-26T05:53:59.401659+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-25T05:59:09.036 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 podman[40889]: 2024-05-25 05:59:08.588600798 +0000 UTC m=+2.668232414 container init 31b85a1d8b12426ada30191ebda46a839ab0b91a17f3fafb0b30f7bdaa99fb42 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T05:59:09.036 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 podman[40889]: 2024-05-25 05:59:08.591431748 +0000 UTC m=+2.671063375 container start 31b85a1d8b12426ada30191ebda46a839ab0b91a17f3fafb0b30f7bdaa99fb42 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T05:59:09.036 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.595Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-25T05:59:09.036 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.595Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=arp 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=edac 2024-05-25T05:59:09.037 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-25T05:59:09.038 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=os 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=stat 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=time 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-25T05:59:09.039 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=uname 2024-05-25T05:59:09.040 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-25T05:59:09.040 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-25T05:59:09.040 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-25T05:59:09.040 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.596Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-25T05:59:09.040 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[40938]: ts=2024-05-25T05:59:08.597Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-25T05:59:09.040 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 bash[40889]: 31b85a1d8b12426ada30191ebda46a839ab0b91a17f3fafb0b30f7bdaa99fb42 2024-05-25T05:59:09.040 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 05:59:08 smithi179 systemd[1]: Started Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:59:09.657 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-25T05:59:09.657 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-25T05:59:09.658 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-25T05:59:09.658 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-25T05:59:09.658 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-25T05:59:09.659 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-25T05:59:09.659 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-25T05:59:09.660 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-25T05:59:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:09 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:09 smithi179 ceph-mon[25623]: Deploying daemon alertmanager.a on smithi175 2024-05-25T05:59:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:09 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2291364680' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:59:10.054 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:09 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:10.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:09 smithi175 ceph-mon[24776]: Deploying daemon alertmanager.a on smithi175 2024-05-25T05:59:10.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:09 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2291364680' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:59:10.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:09 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:10.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:09 smithi175 ceph-mon[29436]: Deploying daemon alertmanager.a on smithi175 2024-05-25T05:59:10.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:09 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2291364680' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-25T05:59:10.077 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:10.126 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:10.126 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:10.127 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:10.128 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:10.131 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:10 smithi179 ceph-mon[25623]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:11.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:10 smithi175 ceph-mon[29436]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:11.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:10 smithi175 ceph-mon[24776]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:11.073 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:12.989 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:12 smithi179 ceph-mon[25623]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:12.990 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:12 smithi175 ceph-mon[29436]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:12.991 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:12 smithi175 ceph-mon[24776]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:14 smithi179 ceph-mon[25623]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:14 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:14.536 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:14 smithi175 ceph-mon[29436]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:14.536 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:14 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:14.536 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:14 smithi175 ceph-mon[24776]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:14.536 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:14 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:15.117 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:59:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:14] "GET /metrics HTTP/1.1" 200 191514 "" "Prometheus/2.33.4" 2024-05-25T05:59:15.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:59:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T05:59:16.634 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:16 smithi175 ceph-mon[29436]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:16.634 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:16 smithi175 ceph-mon[24776]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:16 smithi179 ceph-mon[25623]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:18.680 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:18 smithi175 ceph-mon[24776]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:18.680 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:18 smithi175 ceph-mon[29436]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:18 smithi179 ceph-mon[25623]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:20.644 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:20 smithi175 ceph-mon[24776]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:20.645 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:20 smithi175 ceph-mon[29436]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:20 smithi179 ceph-mon[25623]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:22.122 INFO:teuthology.orchestra.run.smithi175.stdout:77309411362 2024-05-25T05:59:22.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-25T05:59:22.763 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:22 smithi175 ceph-mon[29436]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:22.764 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:22 smithi175 ceph-mon[24776]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:22 smithi179 ceph-mon[25623]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:23.550 INFO:teuthology.orchestra.run.smithi175.stdout:111669149726 2024-05-25T05:59:23.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-25T05:59:24.764 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:24 smithi175 ceph-mon[29436]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:24.764 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:24 smithi175 ceph-mon[24776]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:24 smithi179 ceph-mon[25623]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:25.012 INFO:teuthology.orchestra.run.smithi175.stdout:167503724566 2024-05-25T05:59:25.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-25T05:59:25.034 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:59:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:24] "GET /metrics HTTP/1.1" 200 191514 "" "Prometheus/2.33.4" 2024-05-25T05:59:25.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:59:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T05:59:25.892 INFO:teuthology.orchestra.run.smithi175.stdout:55834574890 2024-05-25T05:59:25.893 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-25T05:59:26.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:26 smithi175 ceph-mon[24776]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:26.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:26 smithi175 ceph-mon[29436]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:26.751 INFO:teuthology.orchestra.run.smithi175.stdout:34359738413 2024-05-25T05:59:26.751 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-25T05:59:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:26 smithi179 ceph-mon[25623]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:28.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:28 smithi175 ceph-mon[24776]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:28 smithi175 ceph-mon[29436]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:28 smithi179 ceph-mon[25623]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:28.882 INFO:teuthology.orchestra.run.smithi175.stdout:133143986204 2024-05-25T05:59:28.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-25T05:59:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:30 smithi175 ceph-mon[24776]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:30 smithi175 ceph-mon[29436]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:30 smithi179 ceph-mon[25623]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:30.869 INFO:teuthology.orchestra.run.smithi175.stdout:236223201293 2024-05-25T05:59:30.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-25T05:59:30.899 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:30.899 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:30.903 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:31.068 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:31.072 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:31.072 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:32.707 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:32 smithi175 ceph-mon[24776]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:32.707 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:32 smithi175 ceph-mon[29436]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:32 smithi179 ceph-mon[25623]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:33.339 INFO:teuthology.orchestra.run.smithi175.stdout:201863462932 2024-05-25T05:59:33.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-25T05:59:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:34 smithi179 ceph-mon[25623]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:34.803 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:34 smithi175 ceph-mon[24776]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:34.803 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:34 smithi175 ceph-mon[29436]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:35.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:59:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:34] "GET /metrics HTTP/1.1" 200 191500 "" "Prometheus/2.33.4" 2024-05-25T05:59:35.551 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:59:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T05:59:36.422 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:36 smithi175 ceph-mon[29436]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:36.422 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:36 smithi175 ceph-mon[24776]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:36 smithi179 ceph-mon[25623]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:38.218 INFO:teuthology.orchestra.run.smithi175.stdout:55834574892 2024-05-25T05:59:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:38 smithi179 ceph-mon[25623]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:38.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:38 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1087328387' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-25T05:59:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:38 smithi175 ceph-mon[24776]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:38 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1087328387' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-25T05:59:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:38 smithi175 ceph-mon[29436]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:38 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1087328387' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-25T05:59:39.281 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:40.728 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:40 smithi175 ceph-mon[29436]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:40.728 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:40 smithi175 ceph-mon[24776]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:40 smithi179 ceph-mon[25623]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:41.255 INFO:teuthology.orchestra.run.smithi175.stdout:77309411366 2024-05-25T05:59:41.338 INFO:teuthology.orchestra.run.smithi175.stdout:236223201295 2024-05-25T05:59:41.632 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:41 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1756310063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-25T05:59:41.632 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:41 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/529332330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-25T05:59:41.632 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:41 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1756310063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-25T05:59:41.632 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:41 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/529332330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-25T05:59:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:41 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1756310063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-25T05:59:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:41 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/529332330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-25T05:59:41.834 INFO:teuthology.orchestra.run.smithi175.stdout:167503724570 2024-05-25T05:59:42.033 INFO:teuthology.orchestra.run.smithi175.stdout:111669149731 2024-05-25T05:59:42.713 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:42 smithi175 ceph-mon[29436]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:42.713 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:42 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/754071286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-25T05:59:42.713 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:42 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3936742464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-25T05:59:42.713 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:42 smithi175 ceph-mon[24776]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:42.714 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:42 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/754071286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-25T05:59:42.714 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:42 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3936742464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-25T05:59:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:42 smithi179 ceph-mon[25623]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:42 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/754071286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-25T05:59:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:42 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3936742464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-25T05:59:43.566 INFO:teuthology.orchestra.run.smithi175.stdout:133143986207 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:59:43.567 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:59:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:59:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T05:59:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:59:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T05:59:43.870 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574892 for osd.1 2024-05-25T05:59:43.870 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:44.585 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201293 got 236223201295 for osd.7 2024-05-25T05:59:44.586 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:44 smithi179 ceph-mon[25623]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:44 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3656675191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-25T05:59:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:44 smithi175 ceph-mon[24776]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:44 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3656675191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-25T05:59:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:44 smithi175 ceph-mon[29436]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:44 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3656675191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-25T05:59:45.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:59:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:44] "GET /metrics HTTP/1.1" 200 191500 "" "Prometheus/2.33.4" 2024-05-25T05:59:45.291 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411362 got 77309411366 for osd.2 2024-05-25T05:59:45.292 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:45.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:59:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T05:59:46.464 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986207 for osd.4 2024-05-25T05:59:46.464 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:46 smithi179 ceph-mon[25623]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:46.804 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:46 smithi175 ceph-mon[24776]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:46.804 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:46 smithi175 ceph-mon[29436]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:46.843 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724566 got 167503724570 for osd.5 2024-05-25T05:59:46.843 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:47.460 INFO:teuthology.orchestra.run.smithi175.stdout:201863462935 2024-05-25T05:59:47.778 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149726 got 111669149731 for osd.3 2024-05-25T05:59:47.778 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:48.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:47 smithi175 podman[46871]: 2024-05-25 05:59:47.965799464 +0000 UTC m=+3.353228726 container create ec0ce44b8f1ca3e1ed5a0cabf012d29351a11e3e114203ad4bb7b1e19e59c265 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:48.423 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 podman[46871]: 2024-05-25 05:59:48.251686034 +0000 UTC m=+3.639115259 container init ec0ce44b8f1ca3e1ed5a0cabf012d29351a11e3e114203ad4bb7b1e19e59c265 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 podman[46871]: 2024-05-25 05:59:48.254593678 +0000 UTC m=+3.642022900 container start ec0ce44b8f1ca3e1ed5a0cabf012d29351a11e3e114203ad4bb7b1e19e59c265 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.265Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.265Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.266Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.175 port=9094 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.267Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.292Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.293Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.295Z caller=main.go:518 msg=Listening address=:9093 2024-05-25T05:59:48.424 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:48.295Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-25T05:59:48.425 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 bash[46871]: ec0ce44b8f1ca3e1ed5a0cabf012d29351a11e3e114203ad4bb7b1e19e59c265 2024-05-25T05:59:48.425 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:48 smithi175 systemd[1]: Started Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T05:59:48.703 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[29436]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:48.703 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1624188443' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-25T05:59:48.704 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:48.704 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:59:48.704 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:59:48.704 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:59:48.704 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[24776]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:48.705 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1624188443' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-25T05:59:48.705 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:48.705 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:59:48.705 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:59:48.705 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:48 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:59:48.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:48 smithi179 ceph-mon[25623]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:48.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:48 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1624188443' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-25T05:59:48.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:48 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:48.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:48 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T05:59:48.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:48 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T05:59:48.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:48 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T05:59:48.839 INFO:teuthology.orchestra.run.smithi175.stdout:34359738418 2024-05-25T05:59:49.540 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:49 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2573622661' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-25T05:59:49.540 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:49 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2573622661' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-25T05:59:49.562 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462932 got 201863462935 for osd.6 2024-05-25T05:59:49.562 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:49.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:49 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2573622661' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-25T05:59:49.999 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738413 got 34359738418 for osd.0 2024-05-25T05:59:49.999 DEBUG:teuthology.parallel:result is None 2024-05-25T05:59:49.999 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-25T05:59:49.999 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph pg dump --format=json 2024-05-25T05:59:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:50 smithi175 ceph-mon[24776]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:50 smithi175 ceph-mon[29436]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:50.571 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:50.268Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000762501s 2024-05-25T05:59:50.626 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:50.781 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:50 smithi179 ceph-mon[25623]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:52.505 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:59:52.505 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:52 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:52.506 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:52 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:52.507 INFO:teuthology.orchestra.run.smithi175.stderr:dumped all 2024-05-25T05:59:52.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:52 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:53.480 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:53 smithi175 ceph-mon[29436]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:53.480 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:53 smithi175 ceph-mon[29436]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-25T05:59:53.480 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:53 smithi175 ceph-mon[24776]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:53.481 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:53 smithi175 ceph-mon[24776]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-25T05:59:53.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:53 smithi179 ceph-mon[25623]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:53.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:53 smithi179 ceph-mon[25623]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-25T05:59:53.864 INFO:teuthology.orchestra.run.smithi175.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-05-25T05:59:51.142876+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48976,"kb_used_data":6160,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912240,"statfs":{"total":767960285184,"available":767910133760,"internally_reserved":0,"allocated":6307840,"data_stored":3510706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002328"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-25T05:58:43.199229+0000","last_change":"2024-05-25T05:58:31.909954+0000","last_active":"2024-05-25T05:58:43.199229+0000","last_peered":"2024-05-25T05:58:43.199229+0000","last_clean":"2024-05-25T05:58:43.199229+0000","last_became_active":"2024-05-25T05:58:31.909491+0000","last_became_peered":"2024-05-25T05:58:31.909491+0000","last_unstale":"2024-05-25T05:58:43.199229+0000","last_undegraded":"2024-05-25T05:58:43.199229+0000","last_fullsized":"2024-05-25T05:58:43.199229+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-25T05:56:32.961357+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-25T05:56:32.961357+0000","last_clean_scrub_stamp":"2024-05-25T05:56:32.961357+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-26T05:56:52.784544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201298,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.313},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.38200000000000001}]},{"osd":3,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.436}]},{"osd":4,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.34300000000000003}]},{"osd":5,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.40300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.628},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":17.968,"5min":17.968,"15min":17.968},"last":0.57699999999999996}]},{"osd":1,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.54700000000000004}]},{"osd":3,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.60299999999999998}]},{"osd":4,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.435}]},{"osd":5,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.501}]},{"osd":7,"last update":"Sat May 25 05:59:30 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.51500000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:11 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.35699999999999998,"15min":0.33900000000000002},"min":{"1min":0.245,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.52800000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.374,"15min":0.35799999999999998},"min":{"1min":0.28899999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.53800000000000003,"5min":0.749,"15min":0.749},"last":0.30399999999999999}]},{"osd":2,"last update":"Sat May 25 05:59:33 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.39000000000000001,"15min":0.38600000000000001},"min":{"1min":0.28199999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.56899999999999995,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.314},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41799999999999998,"15min":0.41499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61499999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat May 25 05:58:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.439,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.29099999999999998}]},{"osd":4,"last update":"Sat May 25 05:59:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.40500000000000003,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61199999999999999,"15min":0.61099999999999999},"min":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82999999999999996,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.68899999999999995}]},{"osd":5,"last update":"Sat May 25 05:59:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.77300000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.67800000000000005}]},{"osd":7,"last update":"Sat May 25 05:59:33 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.69399999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat May 25 05:59:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.378,"15min":0.36599999999999999},"min":{"1min":0.248,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.38500000000000001,"15min":0.37},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat May 25 05:59:41 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.434,"15min":0.434},"min":{"1min":0.28100000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.44800000000000001,"15min":0.45200000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.61699999999999999,"5min":0.627,"15min":0.627},"last":0.38200000000000001}]},{"osd":3,"last update":"Sat May 25 05:59:03 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.46700000000000003,"15min":0.46600000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.503},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.60299999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.32700000000000001}]},{"osd":4,"last update":"Sat May 25 05:59:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58599999999999997,"15min":0.58499999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58199999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat May 25 05:59:41 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.622,"5min":0.61599999999999999,"15min":0.61499999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat May 25 05:59:04 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.436}]},{"osd":7,"last update":"Sat May 25 05:59:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.39800000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":239996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:46 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38200000000000001,"15min":0.375},"min":{"1min":0.26300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39800000000000002,"15min":0.39100000000000001},"min":{"1min":0.24099999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.52400000000000002}]},{"osd":1,"last update":"Sat May 25 05:59:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39900000000000002,"15min":0.38900000000000001},"min":{"1min":0.28699999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53900000000000003,"5min":0.747,"15min":0.747},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.40300000000000002,"15min":0.39300000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.53200000000000003,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat May 25 05:59:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.44400000000000001,"15min":0.436},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.56499999999999995}]},{"osd":4,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.72599999999999998,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.77700000000000002,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.73499999999999999}]},{"osd":5,"last update":"Sat May 25 05:59:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.58899999999999997},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.628,"15min":0.626},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.89000000000000001,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.89000000000000001}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.87},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.70099999999999996}]},{"osd":7,"last update":"Sat May 25 05:59:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.70999999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149733,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:58:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.39500000000000002,"15min":0.39900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.56000000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.43099999999999999,"15min":0.437},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.55000000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.28199999999999997}]},{"osd":1,"last update":"Sat May 25 05:58:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.43099999999999999,"15min":0.434},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.58699999999999997,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.311},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.63300000000000001,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat May 25 05:58:56 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.61699999999999999,"5min":0.62,"15min":0.62},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat May 25 05:59:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73099999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.499},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.59999999999999998,"15min":0.60899999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.79700000000000004,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat May 25 05:59:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.75900000000000001,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56699999999999995,"15min":0.56499999999999995},"min":{"1min":0.41199999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.77800000000000002,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat May 25 05:59:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.437},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat May 25 05:59:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.55900000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.67300000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.68799999999999994,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.65800000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.65700000000000003}]},{"osd":2,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38200000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70799999999999996,"5min":0.755,"15min":0.755},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.64000000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.48199999999999998}]},{"osd":3,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58399999999999996,"15min":0.58699999999999997},"min":{"1min":0.42199999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69599999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.435,"5min":0.377,"15min":0.377},"max":{"1min":0.70699999999999996,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat May 25 05:59:45 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.39300000000000002,"15min":0.377},"min":{"1min":0.317,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.38700000000000001,"15min":0.36699999999999999},"min":{"1min":0.33100000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat May 25 05:59:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.59799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.55600000000000005,"15min":0.56599999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68200000000000005,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.67400000000000004,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.52100000000000002}]},{"osd":1,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56499999999999995,"15min":0.57399999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.623,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.53400000000000003,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.626,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53500000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.78100000000000003,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71199999999999997,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.498}]},{"osd":3,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.56399999999999995,"15min":0.56499999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.81599999999999995,"5min":0.873,"15min":0.873},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.57999999999999996,"15min":0.58799999999999997},"min":{"1min":0.41399999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.65500000000000003,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.54400000000000004}]},{"osd":4,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.41599999999999998,"15min":0.41199999999999998},"min":{"1min":0.34300000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.624,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.42999999999999999,"15min":0.41999999999999998},"min":{"1min":0.32500000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat May 25 05:59:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.48099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-25T05:59:53.866 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph pg dump --format=json 2024-05-25T05:59:54.507 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:54 smithi175 ceph-mon[24776]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:54.507 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:54 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:54.508 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:54 smithi175 ceph-mon[29436]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:54.508 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:54 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:54.510 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:54.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:54 smithi179 ceph-mon[25623]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:54.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:54 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 05:59:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:54] "GET /metrics HTTP/1.1" 200 191500 "" "Prometheus/2.33.4" 2024-05-25T05:59:55.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 05:59:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:05:59:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T05:59:55.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T05:59:55.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T05:59:55.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:55 smithi179 ceph-mon[25623]: Deploying daemon grafana.a on smithi179 2024-05-25T05:59:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T05:59:55.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T05:59:55.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[24776]: Deploying daemon grafana.a on smithi179 2024-05-25T05:59:55.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T05:59:55.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T05:59:55.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T05:59:55.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:55 smithi175 ceph-mon[29436]: Deploying daemon grafana.a on smithi179 2024-05-25T05:59:56.310 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:59:56.311 INFO:teuthology.orchestra.run.smithi175.stderr:dumped all 2024-05-25T05:59:56.980 INFO:teuthology.orchestra.run.smithi175.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-05-25T05:59:55.143657+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48976,"kb_used_data":6160,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912240,"statfs":{"total":767960285184,"available":767910133760,"internally_reserved":0,"allocated":6307840,"data_stored":3510706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002382"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-25T05:58:43.199229+0000","last_change":"2024-05-25T05:58:31.909954+0000","last_active":"2024-05-25T05:58:43.199229+0000","last_peered":"2024-05-25T05:58:43.199229+0000","last_clean":"2024-05-25T05:58:43.199229+0000","last_became_active":"2024-05-25T05:58:31.909491+0000","last_became_peered":"2024-05-25T05:58:31.909491+0000","last_unstale":"2024-05-25T05:58:43.199229+0000","last_undegraded":"2024-05-25T05:58:43.199229+0000","last_fullsized":"2024-05-25T05:58:43.199229+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-25T05:56:32.961357+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-25T05:56:32.961357+0000","last_clean_scrub_stamp":"2024-05-25T05:56:32.961357+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-26T05:56:52.784544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.52500000000000002}]},{"osd":2,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.5},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.51100000000000001}]},{"osd":4,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat May 25 05:59:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.34999999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":17.968,"5min":17.968,"15min":17.968},"last":0.67900000000000005}]},{"osd":1,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57099999999999995}]},{"osd":2,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.59699999999999998}]},{"osd":3,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.66600000000000004}]},{"osd":4,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.60299999999999998}]},{"osd":5,"last update":"Sat May 25 05:59:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.626}]},{"osd":7,"last update":"Sat May 25 05:59:30 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.61799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:11 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.35699999999999998,"15min":0.33900000000000002},"min":{"1min":0.245,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.52800000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.374,"15min":0.35799999999999998},"min":{"1min":0.28899999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.53800000000000003,"5min":0.749,"15min":0.749},"last":0.38}]},{"osd":2,"last update":"Sat May 25 05:59:33 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.39000000000000001,"15min":0.38600000000000001},"min":{"1min":0.28199999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.56899999999999995,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41799999999999998,"15min":0.41499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61499999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.312}]},{"osd":3,"last update":"Sat May 25 05:58:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.439,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat May 25 05:59:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.40500000000000003,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61199999999999999,"15min":0.61099999999999999},"min":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82999999999999996,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat May 25 05:59:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.628},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.77300000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.53800000000000003}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat May 25 05:59:33 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.53100000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat May 25 05:59:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.378,"15min":0.36599999999999999},"min":{"1min":0.248,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.38500000000000001,"15min":0.37},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat May 25 05:59:41 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.434,"15min":0.434},"min":{"1min":0.28100000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.44800000000000001,"15min":0.45200000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.61699999999999999,"5min":0.627,"15min":0.627},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat May 25 05:59:03 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.46700000000000003,"15min":0.46600000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.60299999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat May 25 05:59:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58599999999999997,"15min":0.58499999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58199999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat May 25 05:59:41 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.61599999999999999,"15min":0.61499999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat May 25 05:59:04 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.66900000000000004}]},{"osd":7,"last update":"Sat May 25 05:59:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.628},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57299999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":239996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:46 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38200000000000001,"15min":0.375},"min":{"1min":0.26300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39800000000000002,"15min":0.39100000000000001},"min":{"1min":0.24099999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat May 25 05:59:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39900000000000002,"15min":0.38900000000000001},"min":{"1min":0.28699999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53900000000000003,"5min":0.747,"15min":0.747},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.40300000000000002,"15min":0.39300000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.53200000000000003,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.28599999999999998}]},{"osd":3,"last update":"Sat May 25 05:59:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.44400000000000001,"15min":0.436},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.316}]},{"osd":4,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.72599999999999998,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.501},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.77700000000000002,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.627}]},{"osd":5,"last update":"Sat May 25 05:59:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.58899999999999997},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.628,"15min":0.626},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.89000000000000001,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.52000000000000002}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.60899999999999999}]},{"osd":7,"last update":"Sat May 25 05:59:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65100000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:58:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.39500000000000002,"15min":0.39900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.56000000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.43099999999999999,"15min":0.437},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.55000000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.36899999999999999}]},{"osd":1,"last update":"Sat May 25 05:58:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.43099999999999999,"15min":0.434},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.58699999999999997,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.63300000000000001,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat May 25 05:58:56 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.61699999999999999,"5min":0.62,"15min":0.62},"last":0.31}]},{"osd":4,"last update":"Sat May 25 05:59:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73099999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.59999999999999998,"15min":0.60899999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.79700000000000004,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54000000000000004}]},{"osd":5,"last update":"Sat May 25 05:59:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.75900000000000001,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56699999999999995,"15min":0.56499999999999995},"min":{"1min":0.41199999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.77800000000000002,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat May 25 05:59:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat May 25 05:59:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.5},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.49399999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.67300000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.68799999999999994,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.47599999999999998}]},{"osd":1,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.5},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.65800000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38200000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70799999999999996,"5min":0.755,"15min":0.755},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.64000000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.441}]},{"osd":3,"last update":"Sat May 25 05:59:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58399999999999996,"15min":0.58699999999999997},"min":{"1min":0.42199999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69599999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.435,"5min":0.377,"15min":0.377},"max":{"1min":0.70699999999999996,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.53600000000000003}]},{"osd":5,"last update":"Sat May 25 05:59:45 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.39300000000000002,"15min":0.377},"min":{"1min":0.317,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.38700000000000001,"15min":0.36699999999999999},"min":{"1min":0.33100000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat May 25 05:59:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.433},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.46300000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.55600000000000005,"15min":0.56599999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68200000000000005,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.67400000000000004,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56499999999999995,"15min":0.57399999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.623,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.53400000000000003,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.626,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.66700000000000004}]},{"osd":2,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53500000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.78100000000000003,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71199999999999997,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.68200000000000005}]},{"osd":3,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.56399999999999995,"15min":0.56499999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.81599999999999995,"5min":0.873,"15min":0.873},"last":0.63},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.57999999999999996,"15min":0.58799999999999997},"min":{"1min":0.41399999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.65500000000000003,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.59199999999999997}]},{"osd":4,"last update":"Sat May 25 05:59:43 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.41599999999999998,"15min":0.41199999999999998},"min":{"1min":0.34300000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.624,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.501},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.42999999999999999,"15min":0.41999999999999998},"min":{"1min":0.32500000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat May 25 05:59:07 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat May 25 05:59:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.53500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-25T05:59:56.982 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-25T05:59:56.982 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-25T05:59:56.982 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-25T05:59:56.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- ceph health --format=json 2024-05-25T05:59:57.197 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/mon.a/config 2024-05-25T05:59:57.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:57 smithi179 ceph-mon[25623]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:57.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:57 smithi175 ceph-mon[24776]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:57.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:57 smithi175 ceph-mon[29436]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:58.270 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:58 smithi175 ceph-mon[24776]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-25T05:59:58.271 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:58 smithi175 ceph-mon[29436]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-25T05:59:58.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:58 smithi179 ceph-mon[25623]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-25T05:59:58.570 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 05:59:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T05:59:58.270Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002366128s 2024-05-25T05:59:58.970 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T05:59:58.971 INFO:teuthology.orchestra.run.smithi175.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-25T05:59:59.277 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:59 smithi175 ceph-mon[29436]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:59.278 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 05:59:59 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1207621943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-25T05:59:59.278 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:59 smithi175 ceph-mon[24776]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:59.278 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 05:59:59 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1207621943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-25T05:59:59.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:59 smithi179 ceph-mon[25623]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T05:59:59.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 05:59:59 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1207621943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-25T05:59:59.586 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-25T05:59:59.587 INFO:tasks.cephadm:Setup complete, yielding 2024-05-25T05:59:59.587 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-25T05:59:59.597 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi175.front.sepia.ceph.com 2024-05-25T05:59:59.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-05-25T06:00:00.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:00 smithi175 ceph-mon[29436]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:00.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:00 smithi175 ceph-mon[29436]: overall HEALTH_OK 2024-05-25T06:00:00.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:00 smithi175 ceph-mon[24776]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:00.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:00 smithi175 ceph-mon[24776]: overall HEALTH_OK 2024-05-25T06:00:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:00 smithi179 ceph-mon[25623]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:00 smithi179 ceph-mon[25623]: overall HEALTH_OK 2024-05-25T06:00:02.560 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:00:02.560 INFO:teuthology.orchestra.run.smithi175.stdout: "id": "f7e7c532-4b90-4bbf-b949-24021d834697", 2024-05-25T06:00:02.560 INFO:teuthology.orchestra.run.smithi175.stdout: "name": "r", 2024-05-25T06:00:02.560 INFO:teuthology.orchestra.run.smithi175.stdout: "current_period": "f55e226c-ba98-41d2-ba87-32fa551e2082", 2024-05-25T06:00:02.560 INFO:teuthology.orchestra.run.smithi175.stdout: "epoch": 1 2024-05-25T06:00:02.560 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:00:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:02 smithi179 ceph-mon[25623]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:02 smithi179 ceph-mon[25623]: osdmap e61: 8 total, 8 up, 8 in 2024-05-25T06:00:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:02 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3283582778' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-25T06:00:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:02 smithi175 ceph-mon[24776]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:02 smithi175 ceph-mon[24776]: osdmap e61: 8 total, 8 up, 8 in 2024-05-25T06:00:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:02 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3283582778' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-25T06:00:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:02 smithi175 ceph-mon[29436]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:02 smithi175 ceph-mon[29436]: osdmap e61: 8 total, 8 up, 8 in 2024-05-25T06:00:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:02 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3283582778' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-25T06:00:03.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-05-25T06:00:03.737 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:03 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3283582778' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-25T06:00:03.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:03 smithi175 ceph-mon[29436]: osdmap e62: 8 total, 8 up, 8 in 2024-05-25T06:00:03.738 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:03 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3283582778' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-25T06:00:03.738 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:03 smithi175 ceph-mon[24776]: osdmap e62: 8 total, 8 up, 8 in 2024-05-25T06:00:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:03 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3283582778' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-25T06:00:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:03 smithi179 ceph-mon[25623]: osdmap e62: 8 total, 8 up, 8 in 2024-05-25T06:00:04.779 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:04 smithi175 ceph-mon[24776]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:04.779 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:04 smithi175 ceph-mon[24776]: osdmap e63: 8 total, 8 up, 8 in 2024-05-25T06:00:04.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:04 smithi175 ceph-mon[29436]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:04.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:04 smithi175 ceph-mon[29436]: osdmap e63: 8 total, 8 up, 8 in 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "id": "774e375f-27d7-4208-8881-d45053c8956c", 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "name": "default", 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "api_name": "default", 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "is_master": "true", 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "endpoints": [], 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "hostnames": [], 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "hostnames_s3website": [], 2024-05-25T06:00:04.961 INFO:teuthology.orchestra.run.smithi175.stdout: "master_zone": "", 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout: "zones": [], 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout: "placement_targets": [], 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout: "default_placement": "", 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout: "realm_id": "f7e7c532-4b90-4bbf-b949-24021d834697", 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout: "sync_policy": { 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout: "groups": [] 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:04.962 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:00:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:04 smithi179 ceph-mon[25623]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:04 smithi179 ceph-mon[25623]: osdmap e63: 8 total, 8 up, 8 in 2024-05-25T06:00:05.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:00:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:04] "GET /metrics HTTP/1.1" 200 191504 "" "Prometheus/2.33.4" 2024-05-25T06:00:05.612 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-05-25T06:00:05.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:00:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:00:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:06 smithi175 ceph-mon[24776]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-25T06:00:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:06 smithi175 ceph-mon[29436]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-25T06:00:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:06 smithi179 ceph-mon[25623]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-25T06:00:07.332 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "id": "4ee0da34-3d41-4ce0-95ad-1f56f101f9bd", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "name": "z", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "domain_root": "z.rgw.meta:root", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "control_pool": "z.rgw.control", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "gc_pool": "z.rgw.log:gc", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "lc_pool": "z.rgw.log:lc", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "log_pool": "z.rgw.log", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "roles_pool": "z.rgw.meta:roles", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-05-25T06:00:07.333 INFO:teuthology.orchestra.run.smithi175.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "otp_pool": "z.rgw.otp", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "system_key": { 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "access_key": "", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "secret_key": "" 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "placement_pools": [ 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "key": "default-placement", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "val": { 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "index_pool": "z.rgw.buckets.index", 2024-05-25T06:00:07.334 INFO:teuthology.orchestra.run.smithi175.stdout: "storage_classes": { 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: "STANDARD": { 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: "data_pool": "z.rgw.buckets.data" 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: "index_type": 0 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: "realm_id": "f7e7c532-4b90-4bbf-b949-24021d834697", 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: "notif_pool": "z.rgw.log:notif" 2024-05-25T06:00:07.335 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:00:07.994 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-05-25T06:00:08.566 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:08 smithi175 ceph-mon[29436]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-25T06:00:08.566 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:08 smithi175 ceph-mon[24776]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-25T06:00:08.952 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:08 smithi179 ceph-mon[25623]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-25T06:00:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:10 smithi175 ceph-mon[24776]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s wr, 1 op/s 2024-05-25T06:00:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:10 smithi175 ceph-mon[29436]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s wr, 1 op/s 2024-05-25T06:00:10.869 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:10 smithi179 ceph-mon[25623]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s wr, 1 op/s 2024-05-25T06:00:10.870 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:10 smithi179 systemd[1]: Starting Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:00:11.121 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:10 smithi179 podman[41671]: 2024-05-25 06:00:10.77147115 +0000 UTC m=+0.016199466 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-05-25T06:00:11.122 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:10 smithi179 podman[41671]: 2024-05-25 06:00:10.869591585 +0000 UTC m=+0.114319935 container create c3a0d6b80f616bf3e31340c50a7da211a20a69d60bf98f60d73112ca930be443 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vcs-type=git, build-date=2022-03-28T10:36:18.413762, architecture=x86_64, io.openshift.expose-services=, distribution-scope=public, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, vendor=Red Hat, Inc.) 2024-05-25T06:00:11.122 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 podman[41671]: 2024-05-25 06:00:11.12208834 +0000 UTC m=+0.366816655 container init c3a0d6b80f616bf3e31340c50a7da211a20a69d60bf98f60d73112ca930be443 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, build-date=2022-03-28T10:36:18.413762, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, release=236.1648460182, architecture=x86_64, name=ubi8, io.openshift.expose-services=, description=Ceph Grafana Container, io.openshift.tags=base rhel8, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2) 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 podman[41671]: 2024-05-25 06:00:11.125290311 +0000 UTC m=+0.370018619 container start c3a0d6b80f616bf3e31340c50a7da211a20a69d60bf98f60d73112ca930be443 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, io.buildah.version=1.24.2, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, version=8.5, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, maintainer=Paul Cuzner , description=Ceph Grafana Container, name=ubi8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-25T06:00:11.372 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-25T06:00:11.373 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-05-25T06:00:11.373 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-05-25T06:00:11.373 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-05-25T06:00:11.373 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="App mode production" logger=settings 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-05-25T06:00:11.374 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-05-25T06:00:11.375 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-05-25T06:00:11.376 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-05-25T06:00:11.377 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-05-25T06:00:11.378 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-05-25T06:00:11.379 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-05-25T06:00:11.380 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-05-25T06:00:11.381 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-05-25T06:00:11.381 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-05-25T06:00:11.381 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-05-25T06:00:11.381 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-05-25T06:00:11.381 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 bash[41671]: c3a0d6b80f616bf3e31340c50a7da211a20a69d60bf98f60d73112ca930be443 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 systemd[1]: Started Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-05-25T06:00:11.382 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-05-25T06:00:11.383 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-05-25T06:00:11.384 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-05-25T06:00:11.385 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-05-25T06:00:11.386 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-05-25T06:00:11.387 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-05-25T06:00:11.388 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-05-25T06:00:11.389 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-05-25T06:00:11.639 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:11 smithi179 ceph-mon[25623]: osdmap e64: 8 total, 8 up, 8 in 2024-05-25T06:00:11.639 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:11 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-25T06:00:11.639 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:11 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:11.639 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:11 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:11.639 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:11 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:11.639 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:11 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-05-25T06:00:11.640 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-05-25T06:00:11.641 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-05-25T06:00:11.642 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-05-25T06:00:11.643 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-05-25T06:00:11.644 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-05-25T06:00:11.645 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-05-25T06:00:11.646 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-05-25T06:00:11.647 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-05-25T06:00:11.648 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-05-25T06:00:11.649 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-05-25T06:00:11.650 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-05-25T06:00:11.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-05-25T06:00:11.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-05-25T06:00:11.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-05-25T06:00:11.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=435.584763ms 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-25T06:00:11.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-25T06:00:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[24776]: osdmap e64: 8 total, 8 up, 8 in 2024-05-25T06:00:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-25T06:00:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:11.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[29436]: osdmap e64: 8 total, 8 up, 8 in 2024-05-25T06:00:11.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-25T06:00:11.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:11.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:11.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:11.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:11 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:12.024 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-05-25T06:00:12.024 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-25T06:00:12.024 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-25T06:00:12.025 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-25T06:00:12.025 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:00:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:00:11+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-25T06:00:12.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:12 smithi179 ceph-mon[25623]: pgmap v51: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-25T06:00:12.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:12 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-25T06:00:12.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:12 smithi179 ceph-mon[25623]: osdmap e65: 8 total, 8 up, 8 in 2024-05-25T06:00:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:12 smithi175 ceph-mon[24776]: pgmap v51: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-25T06:00:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:12 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-25T06:00:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:12 smithi175 ceph-mon[24776]: osdmap e65: 8 total, 8 up, 8 in 2024-05-25T06:00:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:12 smithi175 ceph-mon[29436]: pgmap v51: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-25T06:00:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:12 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-25T06:00:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:12 smithi175 ceph-mon[29436]: osdmap e65: 8 total, 8 up, 8 in 2024-05-25T06:00:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:13 smithi179 ceph-mon[25623]: osdmap e66: 8 total, 8 up, 8 in 2024-05-25T06:00:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:13 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-25T06:00:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:13 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:13.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:13 smithi175 ceph-mon[24776]: osdmap e66: 8 total, 8 up, 8 in 2024-05-25T06:00:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:13 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-25T06:00:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:13 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:13 smithi175 ceph-mon[29436]: osdmap e66: 8 total, 8 up, 8 in 2024-05-25T06:00:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:13 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-25T06:00:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:13 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:14 smithi179 ceph-mon[25623]: pgmap v54: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-05-25T06:00:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:14 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-25T06:00:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:14 smithi179 ceph-mon[25623]: osdmap e67: 8 total, 8 up, 8 in 2024-05-25T06:00:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:14 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:14.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[24776]: pgmap v54: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-05-25T06:00:14.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-25T06:00:14.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[24776]: osdmap e67: 8 total, 8 up, 8 in 2024-05-25T06:00:14.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:14.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[29436]: pgmap v54: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-05-25T06:00:14.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1279356910' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-25T06:00:14.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[29436]: osdmap e67: 8 total, 8 up, 8 in 2024-05-25T06:00:14.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:14 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:15.317 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:00:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:14] "GET /metrics HTTP/1.1" 200 195258 "" "Prometheus/2.33.4" 2024-05-25T06:00:15.592 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:00:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:00:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: osdmap e68: 8 total, 8 up, 8 in 2024-05-25T06:00:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-25T06:00:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-25T06:00:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-25T06:00:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: Reconfiguring daemon alertmanager.a on smithi175 2024-05-25T06:00:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-25T06:00:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:15 smithi179 ceph-mon[25623]: osdmap e69: 8 total, 8 up, 8 in 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: osdmap e68: 8 total, 8 up, 8 in 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: Reconfiguring daemon alertmanager.a on smithi175 2024-05-25T06:00:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[24776]: osdmap e69: 8 total, 8 up, 8 in 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: osdmap e68: 8 total, 8 up, 8 in 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: Reconfiguring daemon alertmanager.a on smithi175 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-25T06:00:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:15 smithi175 ceph-mon[29436]: osdmap e69: 8 total, 8 up, 8 in 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[24776]: pgmap v57: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[24776]: osdmap e70: 8 total, 8 up, 8 in 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[29436]: pgmap v57: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-25T06:00:16.855 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-25T06:00:16.856 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-25T06:00:16.856 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[29436]: osdmap e70: 8 total, 8 up, 8 in 2024-05-25T06:00:16.856 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-25T06:00:16.856 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:16 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-25T06:00:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:16 smithi179 ceph-mon[25623]: pgmap v57: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-05-25T06:00:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:16 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-25T06:00:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:16 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-25T06:00:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:16 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-25T06:00:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:16 smithi179 ceph-mon[25623]: osdmap e70: 8 total, 8 up, 8 in 2024-05-25T06:00:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:16 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/142136016' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-25T06:00:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:16 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-25T06:00:17.651 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "id": "8c7fa63e-907d-4828-a6af-f05449a34c15", 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "epoch": 1, 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "predecessor_uuid": "f55e226c-ba98-41d2-ba87-32fa551e2082", 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "sync_status": [], 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "period_map": { 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "id": "8c7fa63e-907d-4828-a6af-f05449a34c15", 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "zonegroups": [ 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "id": "774e375f-27d7-4208-8881-d45053c8956c", 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "name": "default", 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "api_name": "default", 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "is_master": "true", 2024-05-25T06:00:17.652 INFO:teuthology.orchestra.run.smithi175.stdout: "endpoints": [], 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "hostnames": [], 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "hostnames_s3website": [], 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "master_zone": "4ee0da34-3d41-4ce0-95ad-1f56f101f9bd", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "zones": [ 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "id": "4ee0da34-3d41-4ce0-95ad-1f56f101f9bd", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "name": "z", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "endpoints": [], 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "log_meta": "false", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "log_data": "false", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "bucket_index_max_shards": 11, 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "read_only": "false", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "tier_type": "", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "sync_from_all": "true", 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "sync_from": [], 2024-05-25T06:00:17.653 INFO:teuthology.orchestra.run.smithi175.stdout: "redirect_zone": "" 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: "placement_targets": [ 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: "name": "default-placement", 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: "tags": [], 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: "storage_classes": [ 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: "STANDARD" 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: "default_placement": "default-placement", 2024-05-25T06:00:17.654 INFO:teuthology.orchestra.run.smithi175.stdout: "realm_id": "f7e7c532-4b90-4bbf-b949-24021d834697", 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: "sync_policy": { 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: "groups": [] 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: "short_zone_ids": [ 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: "key": "4ee0da34-3d41-4ce0-95ad-1f56f101f9bd", 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: "val": 2735433556 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: "master_zonegroup": "774e375f-27d7-4208-8881-d45053c8956c", 2024-05-25T06:00:17.655 INFO:teuthology.orchestra.run.smithi175.stdout: "master_zone": "4ee0da34-3d41-4ce0-95ad-1f56f101f9bd", 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "period_config": { 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "bucket_quota": { 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "enabled": false, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "check_on_raw": false, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "max_size": -1, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "max_size_kb": 0, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "max_objects": -1 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "user_quota": { 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "enabled": false, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "check_on_raw": false, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "max_size": -1, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "max_size_kb": 0, 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: "max_objects": -1 2024-05-25T06:00:17.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "user_ratelimit": { 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_read_ops": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_write_ops": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_read_bytes": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_write_bytes": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "enabled": false 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "bucket_ratelimit": { 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_read_ops": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_write_ops": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_read_bytes": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_write_bytes": 0, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "enabled": false 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "anonymous_ratelimit": { 2024-05-25T06:00:17.657 INFO:teuthology.orchestra.run.smithi175.stdout: "max_read_ops": 0, 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: "max_write_ops": 0, 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: "max_read_bytes": 0, 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: "max_write_bytes": 0, 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: "enabled": false 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: "realm_id": "f7e7c532-4b90-4bbf-b949-24021d834697", 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: "realm_name": "r", 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout: "realm_epoch": 2 2024-05-25T06:00:17.658 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:00:17.970 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:17 smithi175 systemd[1]: Stopping Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:00:18.312 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-05-25T06:00:18.570 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:18 smithi175 bash[49105]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager.a" found: no such container 2024-05-25T06:00:18.570 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[46965]: level=info ts=2024-05-25T06:00:18.417Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:00:18.570 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:18 smithi175 podman[49117]: 2024-05-25 06:00:18.457794143 +0000 UTC m=+0.164323214 container died ec0ce44b8f1ca3e1ed5a0cabf012d29351a11e3e114203ad4bb7b1e19e59c265 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:18.849 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:18 smithi175 ceph-mon[29436]: pgmap v60: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-05-25T06:00:18.849 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:18 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-25T06:00:18.849 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:18 smithi175 ceph-mon[29436]: osdmap e71: 8 total, 8 up, 8 in 2024-05-25T06:00:18.850 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:18 smithi175 podman[49117]: 2024-05-25 06:00:18.825759682 +0000 UTC m=+0.532288753 container cleanup ec0ce44b8f1ca3e1ed5a0cabf012d29351a11e3e114203ad4bb7b1e19e59c265 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:18.850 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:18 smithi175 bash[49117]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a 2024-05-25T06:00:18.850 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:18 smithi175 ceph-mon[24776]: pgmap v60: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-05-25T06:00:18.850 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:18 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-25T06:00:18.850 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:18 smithi175 ceph-mon[24776]: osdmap e71: 8 total, 8 up, 8 in 2024-05-25T06:00:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:18 smithi179 ceph-mon[25623]: pgmap v60: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-05-25T06:00:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:18 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-25T06:00:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:18 smithi179 ceph-mon[25623]: osdmap e71: 8 total, 8 up, 8 in 2024-05-25T06:00:19.195 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:18 smithi175 bash[49173]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager.a" found: no such container 2024-05-25T06:00:20.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:19 smithi175 podman[49149]: 2024-05-25 06:00:19.831641896 +0000 UTC m=+1.376545199 container remove ec0ce44b8f1ca3e1ed5a0cabf012d29351a11e3e114203ad4bb7b1e19e59c265 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:20.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:19 smithi175 podman[49149]: 2024-05-25 06:00:19.890398769 +0000 UTC m=+1.435302050 volume remove a9e37d1bacc75c8a13eda9dc166c23babec2bbf0f95dd46fe8042d9e529afdbf 2024-05-25T06:00:20.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:19 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-25T06:00:20.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:19 smithi175 systemd[1]: Stopped Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:00:20.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:19 smithi175 systemd[1]: Starting Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:00:20.693 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:20 smithi175 ceph-mon[29436]: pgmap v62: 129 pgs: 18 unknown, 111 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 672 B/s wr, 1 op/s 2024-05-25T06:00:20.694 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:20 smithi175 ceph-mon[24776]: pgmap v62: 129 pgs: 18 unknown, 111 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 672 B/s wr, 1 op/s 2024-05-25T06:00:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:20 smithi179 ceph-mon[25623]: pgmap v62: 129 pgs: 18 unknown, 111 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 672 B/s wr, 1 op/s 2024-05-25T06:00:21.086 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled rgw.foo update... 2024-05-25T06:00:21.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:20 smithi175 podman[49293]: 2024-05-25 06:00:20.980664334 +0000 UTC m=+0.284684646 volume create 47294d149029b72d85f1b9abd1028d661e3d54d86fddf890ddc1e63927e8465e 2024-05-25T06:00:21.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:20 smithi175 podman[49293]: 2024-05-25 06:00:20.8933196 +0000 UTC m=+0.197339922 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-25T06:00:21.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 podman[49293]: 2024-05-25 06:00:21.072677406 +0000 UTC m=+0.376697714 container create aa07cecebf6fcae15d93952c9cbca8b9022d5418b17bcfa12303e359f9004763 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 podman[49293]: 2024-05-25 06:00:21.642999541 +0000 UTC m=+0.947019855 container init aa07cecebf6fcae15d93952c9cbca8b9022d5418b17bcfa12303e359f9004763 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 podman[49293]: 2024-05-25 06:00:21.646326547 +0000 UTC m=+0.950346864 container start aa07cecebf6fcae15d93952c9cbca8b9022d5418b17bcfa12303e359f9004763 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.656Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.657Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.658Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.175 port=9094 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.659Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.684Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.685Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T06:00:21.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.687Z caller=main.go:518 msg=Listening address=:9093 2024-05-25T06:00:21.821 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:21.687Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-25T06:00:21.821 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 bash[49293]: aa07cecebf6fcae15d93952c9cbca8b9022d5418b17bcfa12303e359f9004763 2024-05-25T06:00:21.821 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:21 smithi175 systemd[1]: Started Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:00:22.127 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph osd pool create foo' 2024-05-25T06:00:22.382 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:22 smithi179 ceph-mon[25623]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:00:22.382 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:22 smithi179 ceph-mon[25623]: Saving service rgw.foo spec with placement count:2 2024-05-25T06:00:22.382 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:22 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:22.382 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:22 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[24776]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[24776]: Saving service rgw.foo spec with placement count:2 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[29436]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[29436]: Saving service rgw.foo spec with placement count:2 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:22.431 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:22 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:23.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:23 smithi179 ceph-mon[25623]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-05-25T06:00:23.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:23 smithi179 ceph-mon[25623]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:00:23.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:23 smithi179 ceph-mon[25623]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:00:23.468 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:23 smithi175 ceph-mon[29436]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-05-25T06:00:23.469 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:23 smithi175 ceph-mon[29436]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:00:23.469 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:23 smithi175 ceph-mon[29436]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:00:23.469 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:23 smithi175 ceph-mon[24776]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-05-25T06:00:23.469 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:23 smithi175 ceph-mon[24776]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:00:23.469 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:23 smithi175 ceph-mon[24776]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:00:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:23.660Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000895934s 2024-05-25T06:00:24.997 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 systemd[1]: Stopping Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:00:24.998 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 bash[42274]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:00:25.111 INFO:teuthology.orchestra.run.smithi175.stderr:pool 'foo' created 2024-05-25T06:00:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:25 smithi179 ceph-mon[25623]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 947 B/s wr, 2 op/s 2024-05-25T06:00:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:25 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/712895967' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.997Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.998Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.999Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-25T06:00:25.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.999Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-25T06:00:25.287 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[40670]: ts=2024-05-25T06:00:24.999Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-25T06:00:25.287 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 podman[42286]: 2024-05-25 06:00:25.008381264 +0000 UTC m=+0.122957381 container died b1d4538f4f2b2b004b7e5805ad1f37107da97f1b7fdb8134cc81432106e6da85 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:25.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:25 smithi175 ceph-mon[24776]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 947 B/s wr, 2 op/s 2024-05-25T06:00:25.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:25 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/712895967' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-25T06:00:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:25 smithi175 ceph-mon[29436]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 947 B/s wr, 2 op/s 2024-05-25T06:00:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:25 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/712895967' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-25T06:00:25.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:00:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:24] "GET /metrics HTTP/1.1" 200 195258 "" "Prometheus/2.33.4" 2024-05-25T06:00:25.665 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 podman[42286]: 2024-05-25 06:00:25.380628827 +0000 UTC m=+0.495204926 container cleanup b1d4538f4f2b2b004b7e5805ad1f37107da97f1b7fdb8134cc81432106e6da85 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:25.665 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 bash[42286]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a 2024-05-25T06:00:25.665 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 bash[42313]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:00:25.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'rbd pool init foo' 2024-05-25T06:00:25.924 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 podman[42301]: 2024-05-25 06:00:25.665293661 +0000 UTC m=+0.653963293 container remove b1d4538f4f2b2b004b7e5805ad1f37107da97f1b7fdb8134cc81432106e6da85 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:25.924 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-25T06:00:25.924 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 systemd[1]: Stopped Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:00:25.924 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 systemd[1]: Starting Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:00:25.924 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 podman[42377]: 2024-05-25 06:00:25.808943369 +0000 UTC m=+0.018602958 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-25T06:00:25.924 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:25 smithi179 podman[42377]: 2024-05-25 06:00:25.924757212 +0000 UTC m=+0.134416852 container create 8c1fe73f54ff905cdd88c8f2b62ca2e2e719852d54ff68793c0c2495a4f4f191 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:26.218 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:26 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/712895967' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-25T06:00:26.218 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:26 smithi179 ceph-mon[25623]: osdmap e72: 8 total, 8 up, 8 in 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 podman[42377]: 2024-05-25 06:00:26.135397688 +0000 UTC m=+0.345057288 container init 8c1fe73f54ff905cdd88c8f2b62ca2e2e719852d54ff68793c0c2495a4f4f191 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 podman[42377]: 2024-05-25 06:00:26.138225406 +0000 UTC m=+0.347885000 container start 8c1fe73f54ff905cdd88c8f2b62ca2e2e719852d54ff68793c0c2495a4f4f191 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.161Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.161Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.161Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.162Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi179 (none))" 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.162Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.162Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.163Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.164Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.166Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.167Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.167Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=697.038µs 2024-05-25T06:00:26.219 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.168Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-25T06:00:26.220 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 bash[42377]: 8c1fe73f54ff905cdd88c8f2b62ca2e2e719852d54ff68793c0c2495a4f4f191 2024-05-25T06:00:26.388 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:26 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/712895967' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-25T06:00:26.389 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:26 smithi175 ceph-mon[29436]: osdmap e72: 8 total, 8 up, 8 in 2024-05-25T06:00:26.389 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:26 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/712895967' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-25T06:00:26.389 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:26 smithi175 ceph-mon[24776]: osdmap e72: 8 total, 8 up, 8 in 2024-05-25T06:00:26.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 systemd[1]: Started Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:00:26.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.373Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-05-25T06:00:26.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.373Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-05-25T06:00:26.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.373Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.716µs wal_replay_duration=205.560209ms total_replay_duration=206.303281ms 2024-05-25T06:00:26.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.375Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-25T06:00:26.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.375Z caller=main.go:947 level=info msg="TSDB started" 2024-05-25T06:00:26.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.375Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-25T06:00:26.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.389Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.391329ms db_storage=611ns remote_storage=1.536µs web_handler=335ns query_engine=744ns scrape=363.703µs scrape_sd=29.231µs notify=25.527µs notify_sd=10.784µs rules=13.73866ms 2024-05-25T06:00:26.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:00:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:00:26.389Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: osdmap e73: 8 total, 8 up, 8 in 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.175:9093"}]: dispatch 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.179:9095"}]: dispatch 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.179:3000"}]: dispatch 2024-05-25T06:00:27.396 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: osdmap e73: 8 total, 8 up, 8 in 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.175:9093"}]: dispatch 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.179:9095"}]: dispatch 2024-05-25T06:00:27.397 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.398 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:00:27.398 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.179:3000"}]: dispatch 2024-05-25T06:00:27.398 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.398 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:27.398 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:27.398 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:27 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: osdmap e73: 8 total, 8 up, 8 in 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.175:9093"}]: dispatch 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.179:9095"}]: dispatch 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.179:3000"}]: dispatch 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:27.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:27 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:28.426 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:00:28.426 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.175:9093"}]: dispatch 2024-05-25T06:00:28.426 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:00:28.427 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.179:9095"}]: dispatch 2024-05-25T06:00:28.427 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:00:28.427 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.179:3000"}]: dispatch 2024-05-25T06:00:28.427 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: osdmap e74: 8 total, 8 up, 8 in 2024-05-25T06:00:28.427 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3562842560' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-25T06:00:28.427 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:28 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.175:9093"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.179:9095"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.179:3000"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: osdmap e74: 8 total, 8 up, 8 in 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3562842560' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-25T06:00:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.175:9093"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.179:9095"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.179:3000"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: osdmap e74: 8 total, 8 up, 8 in 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3562842560' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-25T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:28 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-25T06:00:29.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:29 smithi175 ceph-mon[29436]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:29.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:29 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-25T06:00:29.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:29 smithi175 ceph-mon[29436]: osdmap e75: 8 total, 8 up, 8 in 2024-05-25T06:00:29.445 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:29 smithi175 ceph-mon[24776]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:29.445 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:29 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-25T06:00:29.445 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:29 smithi175 ceph-mon[24776]: osdmap e75: 8 total, 8 up, 8 in 2024-05-25T06:00:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:29 smithi179 ceph-mon[25623]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:29 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-25T06:00:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:29 smithi179 ceph-mon[25623]: osdmap e75: 8 total, 8 up, 8 in 2024-05-25T06:00:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: osdmap e76: 8 total, 8 up, 8 in 2024-05-25T06:00:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-05-25T06:00:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: Saving service rgw.foo spec with placement count:2 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:30 smithi179 ceph-mon[25623]: Deploying daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: osdmap e76: 8 total, 8 up, 8 in 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: Saving service rgw.foo spec with placement count:2 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[29436]: Deploying daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: osdmap e76: 8 total, 8 up, 8 in 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: Saving service rgw.foo spec with placement count:2 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:30 smithi175 ceph-mon[24776]: Deploying daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:00:31.176 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:31 smithi175 ceph-mon[29436]: osdmap e77: 8 total, 8 up, 8 in 2024-05-25T06:00:31.176 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:31 smithi175 ceph-mon[24776]: osdmap e77: 8 total, 8 up, 8 in 2024-05-25T06:00:31.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch apply iscsi foo u p' 2024-05-25T06:00:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:31 smithi179 ceph-mon[25623]: osdmap e77: 8 total, 8 up, 8 in 2024-05-25T06:00:31.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:00:31 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:00:31.662Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002968735s 2024-05-25T06:00:32.436 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:32 smithi179 ceph-mon[25623]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 253 B/s wr, 0 op/s 2024-05-25T06:00:32.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:32 smithi175 ceph-mon[29436]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 253 B/s wr, 0 op/s 2024-05-25T06:00:32.446 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:32 smithi175 ceph-mon[24776]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 253 B/s wr, 0 op/s 2024-05-25T06:00:33.888 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled iscsi.foo update... 2024-05-25T06:00:34.658 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'sleep 180' 2024-05-25T06:00:35.190 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:00:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:34] "GET /metrics HTTP/1.1" 200 197876 "" "Prometheus/2.33.4" 2024-05-25T06:00:35.190 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[29436]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-05-25T06:00:35.191 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[29436]: from='client.24601 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:00:35.191 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[29436]: Saving service iscsi.foo spec with placement count:1 2024-05-25T06:00:35.191 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:35.191 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[24776]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-05-25T06:00:35.191 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[24776]: from='client.24601 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:00:35.191 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[24776]: Saving service iscsi.foo spec with placement count:1 2024-05-25T06:00:35.192 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:34 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:34 smithi179 ceph-mon[25623]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-05-25T06:00:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:34 smithi179 ceph-mon[25623]: from='client.24601 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:00:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:34 smithi179 ceph-mon[25623]: Saving service iscsi.foo spec with placement count:1 2024-05-25T06:00:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:34 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:00:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:00:36.009 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:36 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:36 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:36 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:36 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-25T06:00:36.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:36 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:36.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:36 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:36.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:36 smithi179 ceph-mon[25623]: Deploying daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:00:36.286 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:36.286 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:36.286 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:36.286 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[29436]: Deploying daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:36.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:36 smithi175 ceph-mon[24776]: Deploying daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:00:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:37 smithi179 ceph-mon[25623]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 291 B/s wr, 0 op/s 2024-05-25T06:00:37.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:37 smithi175 ceph-mon[24776]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 291 B/s wr, 0 op/s 2024-05-25T06:00:37.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:37 smithi175 ceph-mon[29436]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 291 B/s wr, 0 op/s 2024-05-25T06:00:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:39 smithi179 ceph-mon[25623]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 254 B/s wr, 0 op/s 2024-05-25T06:00:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:39 smithi175 ceph-mon[24776]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 254 B/s wr, 0 op/s 2024-05-25T06:00:39.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:39 smithi175 ceph-mon[29436]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 254 B/s wr, 0 op/s 2024-05-25T06:00:40.633 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:40 smithi179 ceph-mon[25623]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:40.633 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:40 smithi179 ceph-mon[25623]: osdmap e78: 8 total, 8 up, 8 in 2024-05-25T06:00:40.730 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:40 smithi175 ceph-mon[29436]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:40.730 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:40 smithi175 ceph-mon[29436]: osdmap e78: 8 total, 8 up, 8 in 2024-05-25T06:00:40.733 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:40 smithi175 ceph-mon[24776]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:40.733 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:40 smithi175 ceph-mon[24776]: osdmap e78: 8 total, 8 up, 8 in 2024-05-25T06:00:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:41 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:41 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:41 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:41 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:42.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:41 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:42.955 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:42 smithi175 ceph-mon[29436]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:42.955 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:42 smithi175 ceph-mon[24776]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:42.992 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:42 smithi179 ceph-mon[25623]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:00:43.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:00:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:00:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-25T06:00:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-05-25T06:00:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-25T06:00:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-05-25T06:00:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-25T06:00:44.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:44.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-25T06:00:45.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-05-25T06:00:45.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-05-25T06:00:45.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-25T06:00:45.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: osdmap e79: 8 total, 8 up, 8 in 2024-05-25T06:00:45.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:44 smithi179 ceph-mon[25623]: Checking dashboard <-> RGW credentials 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:00:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:44] "GET /metrics HTTP/1.1" 200 207236 "" "Prometheus/2.33.4" 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: osdmap e79: 8 total, 8 up, 8 in 2024-05-25T06:00:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[29436]: Checking dashboard <-> RGW credentials 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 3.4 KiB/s wr, 148 op/s 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: osdmap e79: 8 total, 8 up, 8 in 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:44 smithi175 ceph-mon[24776]: Checking dashboard <-> RGW credentials 2024-05-25T06:00:45.731 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:00:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:00:45.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: osdmap e80: 8 total, 8 up, 8 in 2024-05-25T06:00:45.977 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.977 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.977 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.977 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:00:45.977 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:00:45.978 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-25T06:00:45.978 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:45.978 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: osdmap e80: 8 total, 8 up, 8 in 2024-05-25T06:00:45.978 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.979 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.979 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:45.979 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:00:45.979 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:00:45.979 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-25T06:00:45.979 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:45 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: osdmap e80: 8 total, 8 up, 8 in 2024-05-25T06:00:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:00:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:00:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-25T06:00:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:45 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:46.955 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:46 smithi175 ceph-mon[29436]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 170 B/s wr, 267 op/s 2024-05-25T06:00:46.955 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:46 smithi175 ceph-mon[29436]: Checking pool "foo" exists for service iscsi.foo 2024-05-25T06:00:46.955 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:46 smithi175 ceph-mon[29436]: Deploying daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:00:46.955 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:46 smithi175 ceph-mon[24776]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 170 B/s wr, 267 op/s 2024-05-25T06:00:46.955 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:46 smithi175 ceph-mon[24776]: Checking pool "foo" exists for service iscsi.foo 2024-05-25T06:00:46.955 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:46 smithi175 ceph-mon[24776]: Deploying daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:00:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:46 smithi179 ceph-mon[25623]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 170 B/s wr, 267 op/s 2024-05-25T06:00:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:46 smithi179 ceph-mon[25623]: Checking pool "foo" exists for service iscsi.foo 2024-05-25T06:00:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:46 smithi179 ceph-mon[25623]: Deploying daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:00:49.004 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:48 smithi179 ceph-mon[25623]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 135 KiB/s rd, 131 B/s wr, 206 op/s 2024-05-25T06:00:49.004 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:48 smithi179 ceph-mon[25623]: mgrmap e20: y(active, since 2m), standbys: x 2024-05-25T06:00:49.004 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:48 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:49.004 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:48 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:49.005 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:48 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:49.005 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:48 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:49.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[24776]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 135 KiB/s rd, 131 B/s wr, 206 op/s 2024-05-25T06:00:49.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[24776]: mgrmap e20: y(active, since 2m), standbys: x 2024-05-25T06:00:49.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:49.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:49.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:49.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:49.028 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[29436]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 135 KiB/s rd, 131 B/s wr, 206 op/s 2024-05-25T06:00:49.028 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[29436]: mgrmap e20: y(active, since 2m), standbys: x 2024-05-25T06:00:49.028 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:49.028 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:49.028 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:49.028 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:48 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:49.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:49 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/281476929' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:00:49.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:49 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/4139394693' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]: dispatch 2024-05-25T06:00:49.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:49 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]: dispatch 2024-05-25T06:00:50.045 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:49 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/281476929' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:00:50.045 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:49 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/4139394693' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]: dispatch 2024-05-25T06:00:50.045 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:49 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]: dispatch 2024-05-25T06:00:50.045 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:49 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/281476929' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:00:50.046 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:49 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/4139394693' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]: dispatch 2024-05-25T06:00:50.046 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:49 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]: dispatch 2024-05-25T06:00:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:50 smithi179 ceph-mon[25623]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 511 B/s wr, 224 op/s; 5 B/s, 0 objects/s recovering 2024-05-25T06:00:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:50 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]': finished 2024-05-25T06:00:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:50 smithi179 ceph-mon[25623]: osdmap e81: 8 total, 8 up, 8 in 2024-05-25T06:00:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:50 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/171207988' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]: dispatch 2024-05-25T06:00:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:50 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]: dispatch 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[29436]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 511 B/s wr, 224 op/s; 5 B/s, 0 objects/s recovering 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]': finished 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[29436]: osdmap e81: 8 total, 8 up, 8 in 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/171207988' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]: dispatch 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]: dispatch 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[24776]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 511 B/s wr, 224 op/s; 5 B/s, 0 objects/s recovering 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2643919840"}]': finished 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[24776]: osdmap e81: 8 total, 8 up, 8 in 2024-05-25T06:00:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/171207988' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]: dispatch 2024-05-25T06:00:51.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:50 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]: dispatch 2024-05-25T06:00:51.914 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:51 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]': finished 2024-05-25T06:00:51.914 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:51 smithi179 ceph-mon[25623]: osdmap e82: 8 total, 8 up, 8 in 2024-05-25T06:00:51.914 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:51 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1387624471' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]: dispatch 2024-05-25T06:00:51.914 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:51 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]: dispatch 2024-05-25T06:00:51.914 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:51 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]': finished 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[29436]: osdmap e82: 8 total, 8 up, 8 in 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1387624471' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]: dispatch 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]: dispatch 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/857453375"}]': finished 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[24776]: osdmap e82: 8 total, 8 up, 8 in 2024-05-25T06:00:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1387624471' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]: dispatch 2024-05-25T06:00:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]: dispatch 2024-05-25T06:00:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:51 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 477 B/s wr, 29 op/s; 7 B/s, 0 objects/s recovering 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]': finished 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: osdmap e83: 8 total, 8 up, 8 in 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: Adding iSCSI gateway http://:@172.21.15.175:5000 to Dashboard 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi175"}]: dispatch 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi175"}]: dispatch 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: Checking dashboard <-> RGW credentials 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1566157025' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2149746729"}]: dispatch 2024-05-25T06:00:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:52 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 477 B/s wr, 29 op/s; 7 B/s, 0 objects/s recovering 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]': finished 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: osdmap e83: 8 total, 8 up, 8 in 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: Adding iSCSI gateway http://:@172.21.15.175:5000 to Dashboard 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:00:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi175"}]: dispatch 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi175"}]: dispatch 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: Checking dashboard <-> RGW credentials 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1566157025' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2149746729"}]: dispatch 2024-05-25T06:00:53.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 477 B/s wr, 29 op/s; 7 B/s, 0 objects/s recovering 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2269993462"}]': finished 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: osdmap e83: 8 total, 8 up, 8 in 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: Adding iSCSI gateway http://:@172.21.15.175:5000 to Dashboard 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi175"}]: dispatch 2024-05-25T06:00:53.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi175"}]: dispatch 2024-05-25T06:00:53.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:53.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:00:53.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:00:53.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:00:53.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: Checking dashboard <-> RGW credentials 2024-05-25T06:00:53.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1566157025' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2149746729"}]: dispatch 2024-05-25T06:00:53.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:52 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:54.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1566157025' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2149746729"}]': finished 2024-05-25T06:00:54.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[29436]: osdmap e84: 8 total, 8 up, 8 in 2024-05-25T06:00:54.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:54.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2876619871' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/896188687"}]: dispatch 2024-05-25T06:00:54.245 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1566157025' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2149746729"}]': finished 2024-05-25T06:00:54.246 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[24776]: osdmap e84: 8 total, 8 up, 8 in 2024-05-25T06:00:54.246 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:54.246 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:53 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2876619871' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/896188687"}]: dispatch 2024-05-25T06:00:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:53 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1566157025' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2149746729"}]': finished 2024-05-25T06:00:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:53 smithi179 ceph-mon[25623]: osdmap e84: 8 total, 8 up, 8 in 2024-05-25T06:00:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:53 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:53 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2876619871' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/896188687"}]: dispatch 2024-05-25T06:00:55.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:54 smithi179 ceph-mon[25623]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:55.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:54 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2876619871' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/896188687"}]': finished 2024-05-25T06:00:55.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:54 smithi179 ceph-mon[25623]: osdmap e85: 8 total, 8 up, 8 in 2024-05-25T06:00:55.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:54 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1410930680' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]: dispatch 2024-05-25T06:00:55.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:54 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]: dispatch 2024-05-25T06:00:55.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:00:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:54] "GET /metrics HTTP/1.1" 200 207236 "" "Prometheus/2.33.4" 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[29436]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2876619871' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/896188687"}]': finished 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[29436]: osdmap e85: 8 total, 8 up, 8 in 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1410930680' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]: dispatch 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]: dispatch 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[24776]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2876619871' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/896188687"}]': finished 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[24776]: osdmap e85: 8 total, 8 up, 8 in 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1410930680' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]: dispatch 2024-05-25T06:00:55.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:54 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]: dispatch 2024-05-25T06:00:55.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:00:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:00:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:00:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:56 smithi179 ceph-mon[25623]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 19 op/s 2024-05-25T06:00:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:56 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]': finished 2024-05-25T06:00:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:56 smithi179 ceph-mon[25623]: osdmap e86: 8 total, 8 up, 8 in 2024-05-25T06:00:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:56 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2937794010' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/449852730"}]: dispatch 2024-05-25T06:00:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[29436]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 19 op/s 2024-05-25T06:00:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]': finished 2024-05-25T06:00:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[29436]: osdmap e86: 8 total, 8 up, 8 in 2024-05-25T06:00:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2937794010' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/449852730"}]: dispatch 2024-05-25T06:00:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[24776]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 19 op/s 2024-05-25T06:00:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1757996095"}]': finished 2024-05-25T06:00:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[24776]: osdmap e86: 8 total, 8 up, 8 in 2024-05-25T06:00:56.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:56 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2937794010' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/449852730"}]: dispatch 2024-05-25T06:00:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:57 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2937794010' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/449852730"}]': finished 2024-05-25T06:00:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:57 smithi179 ceph-mon[25623]: osdmap e87: 8 total, 8 up, 8 in 2024-05-25T06:00:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:57 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/602424887' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1040639744"}]: dispatch 2024-05-25T06:00:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:57 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2937794010' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/449852730"}]': finished 2024-05-25T06:00:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:57 smithi175 ceph-mon[24776]: osdmap e87: 8 total, 8 up, 8 in 2024-05-25T06:00:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:57 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/602424887' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1040639744"}]: dispatch 2024-05-25T06:00:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:57 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2937794010' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/449852730"}]': finished 2024-05-25T06:00:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:57 smithi175 ceph-mon[29436]: osdmap e87: 8 total, 8 up, 8 in 2024-05-25T06:00:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:57 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/602424887' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1040639744"}]: dispatch 2024-05-25T06:00:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:58 smithi179 ceph-mon[25623]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-25T06:00:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:58 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/602424887' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1040639744"}]': finished 2024-05-25T06:00:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:58 smithi179 ceph-mon[25623]: osdmap e88: 8 total, 8 up, 8 in 2024-05-25T06:00:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:58 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3192213189' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]: dispatch 2024-05-25T06:00:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:58 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]: dispatch 2024-05-25T06:00:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:58 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[29436]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/602424887' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1040639744"}]': finished 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[29436]: osdmap e88: 8 total, 8 up, 8 in 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3192213189' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]: dispatch 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]: dispatch 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[24776]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/602424887' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1040639744"}]': finished 2024-05-25T06:00:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[24776]: osdmap e88: 8 total, 8 up, 8 in 2024-05-25T06:00:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3192213189' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]: dispatch 2024-05-25T06:00:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]: dispatch 2024-05-25T06:00:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:58 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:00:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:59 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]': finished 2024-05-25T06:00:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:59 smithi179 ceph-mon[25623]: osdmap e89: 8 total, 8 up, 8 in 2024-05-25T06:00:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:59 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:00:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:59 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1111954470' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]: dispatch 2024-05-25T06:00:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:00:59 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]: dispatch 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]': finished 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[29436]: osdmap e89: 8 total, 8 up, 8 in 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1111954470' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]: dispatch 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]: dispatch 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3928322074"}]': finished 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[24776]: osdmap e89: 8 total, 8 up, 8 in 2024-05-25T06:00:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:00:59.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1111954470' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]: dispatch 2024-05-25T06:00:59.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:00:59 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]: dispatch 2024-05-25T06:01:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:00 smithi179 ceph-mon[25623]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:00 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]': finished 2024-05-25T06:01:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:00 smithi179 ceph-mon[25623]: osdmap e90: 8 total, 8 up, 8 in 2024-05-25T06:01:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:00 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/673731639' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/393692458"}]: dispatch 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[29436]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]': finished 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[29436]: osdmap e90: 8 total, 8 up, 8 in 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/673731639' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/393692458"}]: dispatch 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[24776]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2269993462"}]': finished 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[24776]: osdmap e90: 8 total, 8 up, 8 in 2024-05-25T06:01:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:00 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/673731639' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/393692458"}]: dispatch 2024-05-25T06:01:01.685 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:01 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/673731639' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/393692458"}]': finished 2024-05-25T06:01:01.685 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:01 smithi175 ceph-mon[29436]: osdmap e91: 8 total, 8 up, 8 in 2024-05-25T06:01:01.685 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:01 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/246887645' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/613585740"}]: dispatch 2024-05-25T06:01:01.686 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:01 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/673731639' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/393692458"}]': finished 2024-05-25T06:01:01.686 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:01 smithi175 ceph-mon[24776]: osdmap e91: 8 total, 8 up, 8 in 2024-05-25T06:01:01.686 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:01 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/246887645' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/613585740"}]: dispatch 2024-05-25T06:01:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:01 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/673731639' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/393692458"}]': finished 2024-05-25T06:01:01.839 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:01 smithi179 ceph-mon[25623]: osdmap e91: 8 total, 8 up, 8 in 2024-05-25T06:01:01.839 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:01 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/246887645' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/613585740"}]: dispatch 2024-05-25T06:01:02.686 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:02 smithi179 ceph-mon[25623]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:02.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:02 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/246887645' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/613585740"}]': finished 2024-05-25T06:01:02.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:02 smithi179 ceph-mon[25623]: osdmap e92: 8 total, 8 up, 8 in 2024-05-25T06:01:02.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:02 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3229414450' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1469117619"}]: dispatch 2024-05-25T06:01:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[29436]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/246887645' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/613585740"}]': finished 2024-05-25T06:01:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[29436]: osdmap e92: 8 total, 8 up, 8 in 2024-05-25T06:01:02.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3229414450' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1469117619"}]: dispatch 2024-05-25T06:01:02.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[24776]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:02.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/246887645' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/613585740"}]': finished 2024-05-25T06:01:02.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[24776]: osdmap e92: 8 total, 8 up, 8 in 2024-05-25T06:01:02.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:02 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3229414450' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1469117619"}]: dispatch 2024-05-25T06:01:03.699 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:03 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3229414450' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1469117619"}]': finished 2024-05-25T06:01:03.699 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:03 smithi179 ceph-mon[25623]: osdmap e93: 8 total, 8 up, 8 in 2024-05-25T06:01:03.699 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:03 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/4014770417' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/857453375"}]: dispatch 2024-05-25T06:01:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:03 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3229414450' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1469117619"}]': finished 2024-05-25T06:01:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:03 smithi175 ceph-mon[24776]: osdmap e93: 8 total, 8 up, 8 in 2024-05-25T06:01:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:03 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/4014770417' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/857453375"}]: dispatch 2024-05-25T06:01:03.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:03 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3229414450' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1469117619"}]': finished 2024-05-25T06:01:03.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:03 smithi175 ceph-mon[29436]: osdmap e93: 8 total, 8 up, 8 in 2024-05-25T06:01:03.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:03 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/4014770417' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/857453375"}]: dispatch 2024-05-25T06:01:03.821 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:01:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:01:03.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:01:03.821 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:01:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:01:03.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:01:04.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:04 smithi179 ceph-mon[25623]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:01:04.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:04 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/4014770417' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/857453375"}]': finished 2024-05-25T06:01:04.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:04 smithi179 ceph-mon[25623]: osdmap e94: 8 total, 8 up, 8 in 2024-05-25T06:01:04.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:04 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2342396593' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3607309314"}]: dispatch 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[24776]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/4014770417' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/857453375"}]': finished 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[24776]: osdmap e94: 8 total, 8 up, 8 in 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2342396593' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3607309314"}]: dispatch 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[29436]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/4014770417' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/857453375"}]': finished 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[29436]: osdmap e94: 8 total, 8 up, 8 in 2024-05-25T06:01:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:04 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2342396593' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3607309314"}]: dispatch 2024-05-25T06:01:05.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:01:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:04] "GET /metrics HTTP/1.1" 200 208179 "" "Prometheus/2.33.4" 2024-05-25T06:01:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:05 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2342396593' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3607309314"}]': finished 2024-05-25T06:01:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:05 smithi179 ceph-mon[25623]: osdmap e95: 8 total, 8 up, 8 in 2024-05-25T06:01:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:05 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/69136758' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1469117619"}]: dispatch 2024-05-25T06:01:05.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:01:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:01:05.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:05 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2342396593' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3607309314"}]': finished 2024-05-25T06:01:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:05 smithi175 ceph-mon[29436]: osdmap e95: 8 total, 8 up, 8 in 2024-05-25T06:01:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:05 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/69136758' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1469117619"}]: dispatch 2024-05-25T06:01:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:05 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2342396593' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3607309314"}]': finished 2024-05-25T06:01:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:05 smithi175 ceph-mon[24776]: osdmap e95: 8 total, 8 up, 8 in 2024-05-25T06:01:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:05 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/69136758' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1469117619"}]: dispatch 2024-05-25T06:01:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:06 smithi179 ceph-mon[25623]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:06 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/69136758' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1469117619"}]': finished 2024-05-25T06:01:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:06 smithi179 ceph-mon[25623]: osdmap e96: 8 total, 8 up, 8 in 2024-05-25T06:01:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:06 smithi175 ceph-mon[24776]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:06 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/69136758' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1469117619"}]': finished 2024-05-25T06:01:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:06 smithi175 ceph-mon[24776]: osdmap e96: 8 total, 8 up, 8 in 2024-05-25T06:01:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:06 smithi175 ceph-mon[29436]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:06 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/69136758' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1469117619"}]': finished 2024-05-25T06:01:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:06 smithi175 ceph-mon[29436]: osdmap e96: 8 total, 8 up, 8 in 2024-05-25T06:01:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:08 smithi179 ceph-mon[25623]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:01:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:08 smithi175 ceph-mon[24776]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:01:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:08 smithi175 ceph-mon[29436]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:01:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:09 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:09.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:09 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:09 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:10 smithi179 ceph-mon[25623]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:01:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:10 smithi175 ceph-mon[24776]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:01:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:10 smithi175 ceph-mon[29436]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:01:12.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:12 smithi179 ceph-mon[25623]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:01:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:12 smithi175 ceph-mon[24776]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:01:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:12 smithi175 ceph-mon[29436]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:01:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:01:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:01:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:01:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:14 smithi179 ceph-mon[25623]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-05-25T06:01:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:14 smithi175 ceph-mon[29436]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-05-25T06:01:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:14 smithi175 ceph-mon[24776]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-05-25T06:01:15.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:01:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:14] "GET /metrics HTTP/1.1" 200 208167 "" "Prometheus/2.33.4" 2024-05-25T06:01:15.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:01:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:01:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:16 smithi179 ceph-mon[25623]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:01:16.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:16 smithi175 ceph-mon[29436]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:01:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:16 smithi175 ceph-mon[24776]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:01:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:18 smithi179 ceph-mon[25623]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-05-25T06:01:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:18 smithi175 ceph-mon[29436]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-05-25T06:01:18.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:18 smithi175 ceph-mon[24776]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-05-25T06:01:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:19 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:19.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:19 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:19 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:20 smithi179 ceph-mon[25623]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:20 smithi175 ceph-mon[24776]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:20 smithi175 ceph-mon[29436]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:22 smithi179 ceph-mon[25623]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:22.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:22 smithi175 ceph-mon[24776]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:22.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:22 smithi175 ceph-mon[29436]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:24 smithi179 ceph-mon[25623]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:24 smithi175 ceph-mon[24776]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:24.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:24 smithi175 ceph-mon[29436]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:25.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:01:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:24] "GET /metrics HTTP/1.1" 200 208167 "" "Prometheus/2.33.4" 2024-05-25T06:01:25.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:01:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:01:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:26 smithi179 ceph-mon[25623]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:26 smithi175 ceph-mon[24776]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:26 smithi175 ceph-mon[29436]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:28 smithi179 ceph-mon[25623]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:28 smithi175 ceph-mon[24776]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:28 smithi175 ceph-mon[29436]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:29 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:29 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:29 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:30 smithi179 ceph-mon[25623]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:30.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:30 smithi175 ceph-mon[29436]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:30 smithi175 ceph-mon[24776]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:32 smithi179 ceph-mon[25623]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:32 smithi175 ceph-mon[24776]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:32.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:32 smithi175 ceph-mon[29436]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:34 smithi179 ceph-mon[25623]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:34.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:34 smithi175 ceph-mon[24776]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:34 smithi175 ceph-mon[29436]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:35.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:01:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:34] "GET /metrics HTTP/1.1" 200 208099 "" "Prometheus/2.33.4" 2024-05-25T06:01:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:01:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:01:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:36 smithi179 ceph-mon[25623]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:36 smithi175 ceph-mon[24776]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:36 smithi175 ceph-mon[29436]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:38 smithi179 ceph-mon[25623]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:38 smithi175 ceph-mon[24776]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:38 smithi175 ceph-mon[29436]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:39 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:39 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:39 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:40 smithi179 ceph-mon[25623]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:40 smithi175 ceph-mon[24776]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:40 smithi175 ceph-mon[29436]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:42.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:42 smithi179 ceph-mon[25623]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:42.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:42 smithi175 ceph-mon[24776]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:42 smithi175 ceph-mon[29436]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:01:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:01:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:01:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:01:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:01:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:44 smithi179 ceph-mon[25623]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:44.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:44 smithi175 ceph-mon[24776]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:44 smithi175 ceph-mon[29436]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:45.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:01:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:44] "GET /metrics HTTP/1.1" 200 208057 "" "Prometheus/2.33.4" 2024-05-25T06:01:45.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:01:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:01:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:46 smithi175 ceph-mon[24776]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:46 smithi175 ceph-mon[29436]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:46 smithi179 ceph-mon[25623]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:48.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:48 smithi175 ceph-mon[24776]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:48.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:48 smithi175 ceph-mon[29436]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:48 smithi179 ceph-mon[25623]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:49.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:49 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:49.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:49 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:49 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:50 smithi175 ceph-mon[24776]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:50.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:50 smithi175 ceph-mon[29436]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:50 smithi179 ceph-mon[25623]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:52 smithi179 ceph-mon[25623]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:01:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:01:52.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:01:52.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:52 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:01:52.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[24776]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:01:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:01:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:01:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:01:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[29436]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:01:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:01:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:01:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:52 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:01:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:54 smithi175 ceph-mon[24776]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:54 smithi175 ceph-mon[29436]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:54 smithi179 ceph-mon[25623]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:55.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:01:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:54] "GET /metrics HTTP/1.1" 200 208057 "" "Prometheus/2.33.4" 2024-05-25T06:01:55.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:01:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:01:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:01:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:56 smithi175 ceph-mon[24776]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:56 smithi175 ceph-mon[29436]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:56 smithi179 ceph-mon[25623]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:01:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:58 smithi175 ceph-mon[24776]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:58 smithi175 ceph-mon[29436]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:58 smithi179 ceph-mon[25623]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:01:59.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:01:59 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:01:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:01:59 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:01:59 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:00 smithi175 ceph-mon[24776]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:00 smithi175 ceph-mon[29436]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:00 smithi179 ceph-mon[25623]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:02.837 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:02 smithi179 ceph-mon[25623]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:03.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:02 smithi175 ceph-mon[24776]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:02 smithi175 ceph-mon[29436]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:04.923 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:04 smithi175 ceph-mon[29436]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:04.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:04 smithi175 ceph-mon[24776]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:04 smithi179 ceph-mon[25623]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:05.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:02:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:04] "GET /metrics HTTP/1.1" 200 208074 "" "Prometheus/2.33.4" 2024-05-25T06:02:05.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:02:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:02:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:06 smithi179 ceph-mon[25623]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:07.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:06 smithi175 ceph-mon[24776]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:07.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:06 smithi175 ceph-mon[29436]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:08 smithi175 ceph-mon[24776]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:08 smithi175 ceph-mon[29436]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:08 smithi179 ceph-mon[25623]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:09 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:10.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:09 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:09 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:10 smithi179 ceph-mon[25623]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:11.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:10 smithi175 ceph-mon[24776]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:11.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:10 smithi175 ceph-mon[29436]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:12 smithi179 ceph-mon[25623]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:13.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:12 smithi175 ceph-mon[24776]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:12 smithi175 ceph-mon[29436]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:14.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:14 smithi175 ceph-mon[24776]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:14.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:14 smithi175 ceph-mon[29436]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:14 smithi179 ceph-mon[25623]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:15.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:02:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:14] "GET /metrics HTTP/1.1" 200 208071 "" "Prometheus/2.33.4" 2024-05-25T06:02:15.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:02:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:02:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:16 smithi179 ceph-mon[25623]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:16 smithi175 ceph-mon[24776]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:16 smithi175 ceph-mon[29436]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:18.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:18 smithi175 ceph-mon[24776]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:18 smithi175 ceph-mon[29436]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:18 smithi179 ceph-mon[25623]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:19 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:19 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:20.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:19 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:20 smithi179 ceph-mon[25623]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:21.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:20 smithi175 ceph-mon[24776]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:20 smithi175 ceph-mon[29436]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:22 smithi179 ceph-mon[25623]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:23.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:22 smithi175 ceph-mon[24776]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:22 smithi175 ceph-mon[29436]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:24.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:24 smithi175 ceph-mon[29436]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:24.924 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:24 smithi175 ceph-mon[24776]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:24 smithi179 ceph-mon[25623]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:25.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:02:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:24] "GET /metrics HTTP/1.1" 200 208071 "" "Prometheus/2.33.4" 2024-05-25T06:02:25.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:02:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:02:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:26 smithi179 ceph-mon[25623]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:27.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:26 smithi175 ceph-mon[24776]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:27.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:26 smithi175 ceph-mon[29436]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:28 smithi175 ceph-mon[29436]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:28 smithi175 ceph-mon[24776]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:28 smithi179 ceph-mon[25623]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:30.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:29 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:29 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:30.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:29 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:30 smithi179 ceph-mon[25623]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:31.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:30 smithi175 ceph-mon[24776]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:30 smithi175 ceph-mon[29436]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:32 smithi179 ceph-mon[25623]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:33.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:32 smithi175 ceph-mon[24776]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:32 smithi175 ceph-mon[29436]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:34.923 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:34 smithi175 ceph-mon[29436]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:34.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:34 smithi175 ceph-mon[24776]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:35.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:34 smithi179 ceph-mon[25623]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:35.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:02:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:34] "GET /metrics HTTP/1.1" 200 208071 "" "Prometheus/2.33.4" 2024-05-25T06:02:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:02:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:02:37.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:36 smithi179 ceph-mon[25623]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:36 smithi175 ceph-mon[29436]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:37.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:36 smithi175 ceph-mon[24776]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:38 smithi175 ceph-mon[24776]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:38 smithi175 ceph-mon[29436]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:38 smithi179 ceph-mon[25623]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:39 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:39 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:39 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:40 smithi179 ceph-mon[25623]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:40 smithi175 ceph-mon[24776]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:40 smithi175 ceph-mon[29436]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:43.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:42 smithi179 ceph-mon[25623]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:43.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:42 smithi175 ceph-mon[29436]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:43.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:42 smithi175 ceph-mon[24776]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:02:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:02:44.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:02:44.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:02:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:02:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:02:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:02:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:02:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:02:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:02:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:02:44.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:02:44.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:44 smithi175 ceph-mon[29436]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:44.924 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:44 smithi175 ceph-mon[24776]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:44 smithi179 ceph-mon[25623]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:45.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:02:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:44] "GET /metrics HTTP/1.1" 200 208084 "" "Prometheus/2.33.4" 2024-05-25T06:02:45.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:02:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:02:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:46 smithi179 ceph-mon[25623]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:46 smithi175 ceph-mon[29436]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:46 smithi175 ceph-mon[24776]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:48.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:48 smithi175 ceph-mon[29436]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:48.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:48 smithi175 ceph-mon[24776]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:48 smithi179 ceph-mon[25623]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:49 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:50.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:49 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:50.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:49 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:02:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:50 smithi179 ceph-mon[25623]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:50 smithi175 ceph-mon[29436]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:50 smithi175 ceph-mon[24776]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:52.783 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[29436]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:52.783 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:02:52.783 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:02:52.784 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:02:52.784 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[24776]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:52.784 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:02:52.784 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:02:52.784 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:02:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:52 smithi179 ceph-mon[25623]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:02:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:02:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:02:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:53 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:02:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:53 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:02:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:53 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:02:55.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:54 smithi175 ceph-mon[29436]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:54 smithi175 ceph-mon[24776]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:55.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:02:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:54] "GET /metrics HTTP/1.1" 200 208084 "" "Prometheus/2.33.4" 2024-05-25T06:02:55.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:54 smithi179 ceph-mon[25623]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:55.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:02:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:02:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:02:57.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:56 smithi175 ceph-mon[29436]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:57.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:56 smithi175 ceph-mon[24776]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:57.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:56 smithi179 ceph-mon[25623]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:02:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:58 smithi175 ceph-mon[24776]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:59.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:58 smithi175 ceph-mon[29436]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:02:59.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:58 smithi179 ceph-mon[25623]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:00.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:02:59 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:02:59 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:02:59 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:01.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:00 smithi175 ceph-mon[24776]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:00 smithi175 ceph-mon[29436]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:01.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:00 smithi179 ceph-mon[25623]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:02 smithi175 ceph-mon[24776]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:02 smithi175 ceph-mon[29436]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:03.087 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:02 smithi179 ceph-mon[25623]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:04 smithi175 ceph-mon[24776]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:04 smithi175 ceph-mon[29436]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:05.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:03:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:04] "GET /metrics HTTP/1.1" 200 208066 "" "Prometheus/2.33.4" 2024-05-25T06:03:05.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:04 smithi179 ceph-mon[25623]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:05.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:03:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:03:07.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:06 smithi175 ceph-mon[24776]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:07.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:06 smithi175 ceph-mon[29436]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:07.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:06 smithi179 ceph-mon[25623]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:09.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:08 smithi179 ceph-mon[25623]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:09.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:08 smithi175 ceph-mon[24776]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:09.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:08 smithi175 ceph-mon[29436]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:10.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:09 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:10.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:09 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:10.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:09 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:11.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:10 smithi179 ceph-mon[25623]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:11.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:10 smithi175 ceph-mon[24776]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:11.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:10 smithi175 ceph-mon[29436]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:13.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:12 smithi179 ceph-mon[25623]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:13.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:12 smithi175 ceph-mon[24776]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:13.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:12 smithi175 ceph-mon[29436]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:15.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:14 smithi179 ceph-mon[25623]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:15.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:14 smithi175 ceph-mon[29436]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:15.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:14 smithi175 ceph-mon[24776]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:15.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:03:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:14] "GET /metrics HTTP/1.1" 200 208019 "" "Prometheus/2.33.4" 2024-05-25T06:03:15.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:03:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:03:17.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:16 smithi179 ceph-mon[25623]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:17.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:16 smithi175 ceph-mon[29436]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:17.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:16 smithi175 ceph-mon[24776]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:18 smithi175 ceph-mon[24776]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:18 smithi175 ceph-mon[29436]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:19.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:18 smithi179 ceph-mon[25623]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:19 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:20.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:19 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:20.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:19 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:20 smithi179 ceph-mon[25623]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:21.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:20 smithi175 ceph-mon[24776]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:20 smithi175 ceph-mon[29436]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:23.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:22 smithi179 ceph-mon[25623]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:23.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:22 smithi175 ceph-mon[24776]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:23.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:22 smithi175 ceph-mon[29436]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:24 smithi179 ceph-mon[25623]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:25.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:24 smithi175 ceph-mon[24776]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:24 smithi175 ceph-mon[29436]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:25.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:03:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:24] "GET /metrics HTTP/1.1" 200 208019 "" "Prometheus/2.33.4" 2024-05-25T06:03:25.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:03:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:03:27.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:26 smithi179 ceph-mon[25623]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:27.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:26 smithi175 ceph-mon[24776]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:26 smithi175 ceph-mon[29436]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:29.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:28 smithi179 ceph-mon[25623]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:29.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:28 smithi175 ceph-mon[29436]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:29.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:28 smithi175 ceph-mon[24776]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:30.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:29 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:30.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:29 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:30.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:29 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:30 smithi179 ceph-mon[25623]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:31.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:30 smithi175 ceph-mon[24776]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:30 smithi175 ceph-mon[29436]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:32 smithi179 ceph-mon[25623]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:33.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:32 smithi175 ceph-mon[24776]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:33.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:32 smithi175 ceph-mon[29436]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:34 smithi179 ceph-mon[25623]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:35.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:34 smithi175 ceph-mon[24776]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:35.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:34 smithi175 ceph-mon[29436]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:35.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:03:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:34] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-05-25T06:03:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:03:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:03:37.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:36 smithi175 ceph-mon[24776]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:36 smithi175 ceph-mon[29436]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:37.225 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-05-25T06:03:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:36 smithi179 ceph-mon[25623]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:38 smithi175 ceph-mon[29436]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:39.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:38 smithi175 ceph-mon[24776]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:38 smithi179 ceph-mon[25623]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:39.803 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-05-25T06:03:40.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:39 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:39 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:40.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:39 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:40 smithi175 ceph-mon[24776]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:40 smithi175 ceph-mon[29436]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:41.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:40 smithi179 ceph-mon[25623]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:42.368 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph config set global log_to_journald false --force' 2024-05-25T06:03:42.987 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:42 smithi175 ceph-mon[29436]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:42.987 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:42 smithi175 ceph-mon[24776]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:43.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:42 smithi179 ceph-mon[25623]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:03:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:03:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:03:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:03:44.315 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:03:44.880 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:03:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:44 smithi175 ceph-mon[29436]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:45.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:44 smithi175 ceph-mon[24776]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:45.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:03:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:44] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-25T06:03:45.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:44 smithi179 ceph-mon[25623]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:45.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:03:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (3m) 2m ago 3m 14.9M - ba2b418f427c aa07cecebf6f 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (3m) 2m ago 3m 45.6M - 8.3.5 dad864ee21e9 c3a0d6b80f61 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (2m) 2m ago 2m 66.9M - 3.5 e1d6a67b021e cbf3ec05b70e 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443 running (8m) 2m ago 8m 410M - 17.2.0 e1d6a67b021e 31d6842a88f3 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:9283 running (10m) 2m ago 10m 454M - 17.2.0 e1d6a67b021e 8822d7e4e7de 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (10m) 2m ago 10m 46.3M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (8m) 2m ago 8m 38.4M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:03:46.714 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (8m) 2m ago 8m 39.8M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (4m) 2m ago 4m 16.4M - 1dbe0e931976 cbca05d03edb 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (4m) 2m ago 4m 15.8M - 1dbe0e931976 31b85a1d8b12 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (8m) 2m ago 8m 50.9M 4016M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (7m) 2m ago 7m 51.7M 4016M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (7m) 2m ago 7m 46.3M 4016M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (6m) 2m ago 6m 45.6M 4016M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (6m) 2m ago 6m 45.9M 4272M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (6m) 2m ago 6m 45.0M 4272M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (5m) 2m ago 5m 46.6M 4272M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (5m) 2m ago 5m 47.9M 4272M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (3m) 2m ago 4m 44.3M - 514e6a882f6e 8c1fe73f54ff 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (3m) 2m ago 3m 69.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:03:46.715 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (3m) 2m ago 3m 72.4M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:03:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:46 smithi179 ceph-mon[25623]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:47.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:46 smithi175 ceph-mon[24776]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:47.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:46 smithi175 ceph-mon[29436]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:47.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions' 2024-05-25T06:03:47.983 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:47 smithi175 ceph-mon[29436]: from='client.24850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:47.983 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:47 smithi175 ceph-mon[24776]: from='client.24850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:47 smithi179 ceph-mon[25623]: from='client.24850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:48.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:48 smithi175 ceph-mon[29436]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:48.942 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:48 smithi175 ceph-mon[24776]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:48 smithi179 ceph-mon[25623]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:49.399 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:03:49.399 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:03:49.400 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:03:49.401 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-05-25T06:03:49.401 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:03:49.401 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:03:50.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph -s' 2024-05-25T06:03:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:49 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:49 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1194167782' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:03:50.316 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:49 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:50.316 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:49 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1194167782' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:03:50.316 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:49 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:03:50.316 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:49 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1194167782' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:03:51.218 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:50 smithi175 ceph-mon[29436]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:51.219 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:50 smithi175 ceph-mon[24776]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:51.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:50 smithi179 ceph-mon[25623]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: cluster: 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: id: 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: health: HEALTH_OK 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: services: 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: mgr: y(active, since 5m), standbys: x 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: data: 2024-05-25T06:03:51.936 INFO:teuthology.orchestra.run.smithi175.stdout: pools: 6 pools, 161 pgs 2024-05-25T06:03:51.937 INFO:teuthology.orchestra.run.smithi175.stdout: objects: 241 objects, 585 KiB 2024-05-25T06:03:51.937 INFO:teuthology.orchestra.run.smithi175.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:03:51.937 INFO:teuthology.orchestra.run.smithi175.stdout: pgs: 161 active+clean 2024-05-25T06:03:51.937 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:03:51.937 INFO:teuthology.orchestra.run.smithi175.stdout: io: 2024-05-25T06:03:51.937 INFO:teuthology.orchestra.run.smithi175.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-25T06:03:51.937 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:03:52.616 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ls' 2024-05-25T06:03:53.103 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:52 smithi179 ceph-mon[25623]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:53.103 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:52 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2137219274' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:03:53.103 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:03:53.103 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:03:53.103 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:52 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:03:53.112 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[24776]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:53.112 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2137219274' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:03:53.112 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:03:53.112 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:03:53.112 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:03:53.112 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[29436]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:53.113 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2137219274' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:03:53.113 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:03:53.113 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:03:53.113 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:52 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:03:54.186 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:54 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:03:54.187 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:54 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:03:54.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:54 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:03:54.560 INFO:teuthology.orchestra.run.smithi175.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-25T06:03:54.560 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi175=a;count:1 2024-05-25T06:03:54.560 INFO:teuthology.orchestra.run.smithi175.stdout:grafana ?:3000 1/1 3m ago 5m smithi179=a;count:1 2024-05-25T06:03:54.560 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-05-25T06:03:54.561 INFO:teuthology.orchestra.run.smithi175.stdout:mgr 2/2 3m ago 8m smithi175=y;smithi179=x;count:2 2024-05-25T06:03:54.561 INFO:teuthology.orchestra.run.smithi175.stdout:mon 3/3 3m ago 9m smithi175:172.21.15.175=a;smithi175:[v2:172.21.15.175:3301,v1:172.21.15.175:6790]=c;smithi179:172.21.15.179=b;count:3 2024-05-25T06:03:54.561 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi175=a;smithi179=b;count:2 2024-05-25T06:03:54.561 INFO:teuthology.orchestra.run.smithi175.stdout:osd 8 3m ago - 2024-05-25T06:03:54.561 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus ?:9095 1/1 3m ago 5m smithi179=a;count:1 2024-05-25T06:03:54.561 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-05-25T06:03:55.190 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:55 smithi175 ceph-mon[24776]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:55.190 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:03:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:54] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-25T06:03:55.191 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:55 smithi175 ceph-mon[29436]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:55.260 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-25T06:03:55.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:55 smithi179 ceph-mon[25623]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:55.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:03:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:03:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:03:56.249 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:56 smithi175 ceph-mon[29436]: from='client.24868 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:56.250 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:56 smithi175 ceph-mon[24776]: from='client.24868 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:56.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:56 smithi179 ceph-mon[25623]: from='client.24868 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:57.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:57 smithi179 ceph-mon[25623]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:57.560 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled to redeploy mgr.x on host 'smithi179' 2024-05-25T06:03:57.560 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:57 smithi175 ceph-mon[24776]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:57.561 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:57 smithi175 ceph-mon[29436]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:03:58.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:58 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1173447907' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T06:03:58.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:58 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:03:58.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:58 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:03:58.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:58 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1173447907' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T06:03:58.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:58 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:03:58.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:58 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:03:58.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps --refresh' 2024-05-25T06:03:58.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:58 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1173447907' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T06:03:58.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:58 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:03:58.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:58 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:03:59.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:59 smithi175 ceph-mon[29436]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:59.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:59 smithi175 ceph-mon[29436]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:59.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:03:59 smithi175 ceph-mon[29436]: Schedule redeploy daemon mgr.x 2024-05-25T06:03:59.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:59 smithi175 ceph-mon[24776]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:59.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:59 smithi175 ceph-mon[24776]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:59.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:03:59 smithi175 ceph-mon[24776]: Schedule redeploy daemon mgr.x 2024-05-25T06:03:59.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:59 smithi179 ceph-mon[25623]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:03:59.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:59 smithi179 ceph-mon[25623]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:03:59.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:03:59 smithi179 ceph-mon[25623]: Schedule redeploy daemon mgr.x 2024-05-25T06:04:00.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:00 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:00.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:00 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:00.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:00 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:00.557 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:04:00.557 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (3m) 3m ago 4m 14.9M - ba2b418f427c aa07cecebf6f 2024-05-25T06:04:00.557 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (3m) 3m ago 3m 45.6M - 8.3.5 dad864ee21e9 c3a0d6b80f61 2024-05-25T06:04:00.557 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (3m) 3m ago 3m 66.9M - 3.5 e1d6a67b021e cbf3ec05b70e 2024-05-25T06:04:00.557 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 31d6842a88f3 2024-05-25T06:04:00.557 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 8822d7e4e7de 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (10m) 3m ago 10m 46.3M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (8m) 3m ago 8m 38.4M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (9m) 3m ago 9m 39.8M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (4m) 3m ago 4m 16.4M - 1dbe0e931976 cbca05d03edb 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (4m) 3m ago 4m 15.8M - 1dbe0e931976 31b85a1d8b12 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (8m) 3m ago 8m 50.9M 4016M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (7m) 3m ago 7m 51.7M 4016M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (7m) 3m ago 7m 46.3M 4016M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (7m) 3m ago 7m 45.6M 4016M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (6m) 3m ago 6m 45.9M 4272M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (6m) 3m ago 6m 45.0M 4272M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (6m) 3m ago 6m 46.6M 4272M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:04:00.558 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (5m) 3m ago 5m 47.9M 4272M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:04:00.559 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (3m) 3m ago 5m 44.3M - 514e6a882f6e 8c1fe73f54ff 2024-05-25T06:04:00.559 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (3m) 3m ago 3m 69.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:04:00.559 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (3m) 3m ago 3m 72.4M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:04:01.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:01 smithi175 ceph-mon[24776]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:01.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:01 smithi175 ceph-mon[29436]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:01.387 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'sleep 180' 2024-05-25T06:04:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:01 smithi179 ceph-mon[25623]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:02.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:02 smithi179 ceph-mon[25623]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:04:02.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:02 smithi175 ceph-mon[29436]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:04:02.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:02 smithi175 ceph-mon[24776]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:04:03.436 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:03 smithi179 ceph-mon[25623]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:03.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:03 smithi175 ceph-mon[24776]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:03.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:03 smithi175 ceph-mon[29436]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:04.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:04 smithi175 ceph-mon[24776]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:04.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:04 smithi175 ceph-mon[29436]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:04.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:04 smithi179 ceph-mon[25623]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:05.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:04:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:04] "GET /metrics HTTP/1.1" 200 208007 "" "Prometheus/2.33.4" 2024-05-25T06:04:05.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:04:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:04:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:06 smithi179 ceph-mon[25623]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:06.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:06 smithi175 ceph-mon[24776]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:06 smithi175 ceph-mon[29436]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:08 smithi179 ceph-mon[25623]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:08.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:08 smithi175 ceph-mon[24776]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:08 smithi175 ceph-mon[29436]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:09 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:09.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:09 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:09 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:10 smithi179 ceph-mon[25623]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:10 smithi175 ceph-mon[24776]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:10 smithi175 ceph-mon[29436]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:12.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:12 smithi179 ceph-mon[25623]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:12.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:12 smithi175 ceph-mon[24776]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:12 smithi175 ceph-mon[29436]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:14 smithi179 ceph-mon[25623]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:14.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:14 smithi175 ceph-mon[24776]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:14.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:14 smithi175 ceph-mon[29436]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:15.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:04:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:14] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-05-25T06:04:15.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:04:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:04:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:16 smithi179 ceph-mon[25623]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:16.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:16 smithi175 ceph-mon[24776]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:16 smithi175 ceph-mon[29436]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:18 smithi179 ceph-mon[25623]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:18.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:18 smithi175 ceph-mon[24776]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:18 smithi175 ceph-mon[29436]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:19 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:19.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:19 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:19 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:20 smithi179 ceph-mon[25623]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:20.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:20 smithi175 ceph-mon[24776]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:20 smithi175 ceph-mon[29436]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:22 smithi179 ceph-mon[25623]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:22.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:22 smithi175 ceph-mon[24776]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:22.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:22 smithi175 ceph-mon[29436]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:24 smithi179 ceph-mon[25623]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:24.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:24 smithi175 ceph-mon[24776]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:24.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:24 smithi175 ceph-mon[29436]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:25.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:04:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:24] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-05-25T06:04:25.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:04:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:04:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:26 smithi179 ceph-mon[25623]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:26.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:26 smithi175 ceph-mon[24776]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:26 smithi175 ceph-mon[29436]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:28 smithi179 ceph-mon[25623]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:28.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:28 smithi175 ceph-mon[24776]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:28 smithi175 ceph-mon[29436]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:29.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:29 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:29 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:29 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:30 smithi179 ceph-mon[25623]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:30.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:30 smithi175 ceph-mon[24776]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:30 smithi175 ceph-mon[29436]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:32 smithi179 ceph-mon[25623]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:32.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:32 smithi175 ceph-mon[24776]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:32.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:32 smithi175 ceph-mon[29436]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:34 smithi179 ceph-mon[25623]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:34.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:34 smithi175 ceph-mon[29436]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:34.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:34 smithi175 ceph-mon[24776]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:35.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:04:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:34] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-05-25T06:04:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:04:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:04:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:36 smithi179 ceph-mon[25623]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:36.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:36 smithi175 ceph-mon[29436]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:36 smithi175 ceph-mon[24776]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:38 smithi179 ceph-mon[25623]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:38 smithi175 ceph-mon[24776]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:38 smithi175 ceph-mon[29436]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:39 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:39.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:39 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:39 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:40 smithi179 ceph-mon[25623]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:40.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:40 smithi175 ceph-mon[24776]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:40 smithi175 ceph-mon[29436]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:42.538 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[29436]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:42.539 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:42.539 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:04:42.539 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:04:42.539 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[24776]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:42.539 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:42.539 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:04:42.539 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:42 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:04:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:42 smithi179 ceph-mon[25623]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:42 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:42.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:42 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:04:42.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:42 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:04:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:04:43.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:04:43.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:43 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:04:43.935 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:43.935 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:04:43.935 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:43 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:04:43.935 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:04:43.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:43 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:04:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:44 smithi179 ceph-mon[25623]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:44.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:44 smithi175 ceph-mon[24776]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:44.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:44 smithi175 ceph-mon[29436]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:45.312 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:04:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:44] "GET /metrics HTTP/1.1" 200 208026 "" "Prometheus/2.33.4" 2024-05-25T06:04:45.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:04:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:04:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:47.052 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.053 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.053 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.053 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:04:47.053 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:04:47.053 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:04:47.053 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:04:47.053 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:46 smithi179 ceph-mon[25623]: Deploying daemon mgr.x on smithi179 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:04:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:04:47.073 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[29436]: Deploying daemon mgr.x on smithi179 2024-05-25T06:04:47.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:47.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:04:47.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:04:47.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:04:47.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:04:47.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:04:47.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:46 smithi175 ceph-mon[24776]: Deploying daemon mgr.x on smithi179 2024-05-25T06:04:48.872 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:48 smithi175 ceph-mon[29436]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:48.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:48 smithi175 ceph-mon[24776]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:48 smithi179 ceph-mon[25623]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:49 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:50.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:49 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:50.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:49 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:04:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:50 smithi179 ceph-mon[25623]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:51.068 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:50 smithi175 ceph-mon[24776]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:51.068 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:50 smithi175 ceph-mon[29436]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:52 smithi179 ceph-mon[25623]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:53.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:52 smithi175 ceph-mon[24776]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:52 smithi175 ceph-mon[29436]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:04:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:04:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:04:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:04:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:04:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:04:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:54 smithi179 ceph-mon[25623]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:55.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:54 smithi175 ceph-mon[24776]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:55.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:54 smithi175 ceph-mon[29436]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:55.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:04:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:54] "GET /metrics HTTP/1.1" 200 208026 "" "Prometheus/2.33.4" 2024-05-25T06:04:55.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:04:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:04:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:04:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:56 smithi179 ceph-mon[25623]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:57.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:56 smithi175 ceph-mon[24776]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:57.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:56 smithi175 ceph-mon[29436]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:04:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:58 smithi179 ceph-mon[25623]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:59.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:58 smithi175 ceph-mon[24776]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:04:59.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:58 smithi175 ceph-mon[29436]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:04:59 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:00.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:04:59 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:04:59 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:00.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:00 smithi175 ceph-mon[24776]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:00 smithi175 ceph-mon[29436]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:00 smithi179 ceph-mon[25623]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:02 smithi179 ceph-mon[25623]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:02 smithi175 ceph-mon[24776]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:02 smithi175 ceph-mon[29436]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:05:03.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:04 smithi179 ceph-mon[25623]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:05.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:04 smithi175 ceph-mon[24776]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:04 smithi175 ceph-mon[29436]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:05.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:05:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:04] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-05-25T06:05:05.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:05:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:06 smithi179 ceph-mon[25623]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:07.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:06 smithi175 ceph-mon[24776]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:07.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:06 smithi175 ceph-mon[29436]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:08 smithi179 ceph-mon[25623]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:09.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:08 smithi175 ceph-mon[24776]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:08 smithi175 ceph-mon[29436]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:09 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:09 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:09 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:10 smithi179 ceph-mon[25623]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:11.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:10 smithi175 ceph-mon[24776]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:11.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:10 smithi175 ceph-mon[29436]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:12 smithi179 ceph-mon[25623]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:13.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:12 smithi175 ceph-mon[24776]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:12 smithi175 ceph-mon[29436]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:05:13.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:14 smithi179 ceph-mon[25623]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:15.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:14 smithi175 ceph-mon[24776]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:14 smithi175 ceph-mon[29436]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:15.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:05:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:14] "GET /metrics HTTP/1.1" 200 208030 "" "Prometheus/2.33.4" 2024-05-25T06:05:15.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[26795]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:05:16.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:16 smithi179 ceph-mon[25623]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:16 smithi175 ceph-mon[24776]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:16 smithi175 ceph-mon[29436]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:18.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:17 smithi179 systemd[1]: Stopping Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:05:18.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 bash[45568]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr.x" found: no such container 2024-05-25T06:05:18.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 podman[45579]: 2024-05-25 06:05:18.14993148 +0000 UTC m=+0.130563233 container died 31d6842a88f319a67eda3c1857487cd0c66677f1f18234e30c66233551550600 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_BRANCH=HEAD, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, name=centos-stream, distribution-scope=public, io.buildah.version=1.19.8, RELEASE=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vendor=Red Hat, Inc.) 2024-05-25T06:05:18.754 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 podman[45579]: 2024-05-25 06:05:18.503809771 +0000 UTC m=+0.484441516 container cleanup 31d6842a88f319a67eda3c1857487cd0c66677f1f18234e30c66233551550600 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., name=centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-05-25T06:05:18.754 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 bash[45579]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x 2024-05-25T06:05:18.754 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 bash[45606]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr.x" found: no such container 2024-05-25T06:05:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:18 smithi179 ceph-mon[25623]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:19.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 podman[45593]: 2024-05-25 06:05:18.763621753 +0000 UTC m=+0.608868730 container remove 31d6842a88f319a67eda3c1857487cd0c66677f1f18234e30c66233551550600 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vendor=Red Hat, Inc., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream) 2024-05-25T06:05:19.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:05:19.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-25T06:05:19.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 systemd[1]: Stopped Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:05:19.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Consumed 17.510s CPU time. 2024-05-25T06:05:19.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:18 smithi179 systemd[1]: Starting Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:05:19.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:18 smithi175 ceph-mon[24776]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:18 smithi175 ceph-mon[29436]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:19.425 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 podman[45708]: 2024-05-25 06:05:19.074812157 +0000 UTC m=+0.017033835 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:05:19.425 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 podman[45708]: 2024-05-25 06:05:19.17328232 +0000 UTC m=+0.115503971 container create 2c455cbdf4573339ce2f9bb9ffc37fffc5426cb4f4aca988d559dc94758b352f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:05:19.764 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 podman[45708]: 2024-05-25 06:05:19.425884543 +0000 UTC m=+0.368106202 container init 2c455cbdf4573339ce2f9bb9ffc37fffc5426cb4f4aca988d559dc94758b352f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:05:19.764 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 podman[45708]: 2024-05-25 06:05:19.428708501 +0000 UTC m=+0.370930161 container start 2c455cbdf4573339ce2f9bb9ffc37fffc5426cb4f4aca988d559dc94758b352f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, RELEASE=squid-ea0684c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-05-25T06:05:19.764 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 ceph-mgr[45727]: -- 172.21.15.179:0/3300009159 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dd44e345a0 con 0x55dd44e09000 2024-05-25T06:05:19.764 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 bash[45708]: 2c455cbdf4573339ce2f9bb9ffc37fffc5426cb4f4aca988d559dc94758b352f 2024-05-25T06:05:19.764 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 systemd[1]: Started Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:05:19.764 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:19.592+0000 7faa45948200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:05:19.906 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:19.906 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:19.906 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:05:19.907 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:19 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:05:20.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:19.788+0000 7faa45948200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:05:20.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:19.904+0000 7faa45948200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:05:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:19 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:19 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:19 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:19 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:05:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:19 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:05:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:19 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:05:20.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:20.246+0000 7faa45948200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:05:20.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:20.331+0000 7faa45948200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:05:21.142 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:20 smithi179 ceph-mon[25623]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:21.142 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:20 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:21.295 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:20 smithi175 ceph-mon[24776]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:21.296 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:20 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:21.296 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:20 smithi175 ceph-mon[29436]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:21.296 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:20 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:21.569 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:21.295Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:05:21.986 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:21.634+0000 7faa45948200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:05:21.986 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:21.825+0000 7faa45948200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:05:22.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:22.009+0000 7faa45948200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:05:22.640 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:22.316+0000 7faa45948200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:05:22.640 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:22.411+0000 7faa45948200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:05:22.890 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:22.639+0000 7faa45948200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:05:22.890 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:22.838+0000 7faa45948200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:05:23.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:22 smithi179 ceph-mon[25623]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:23.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:22 smithi175 ceph-mon[24776]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:23.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:22 smithi175 ceph-mon[29436]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:23.719 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:23.456+0000 7faa45948200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:05:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:05:23.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:23.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:05:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:23.999 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:23.902+0000 7faa45948200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:05:24.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:24.010+0000 7faa45948200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:05:24.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:24.248+0000 7faa45948200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:05:25.136 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:24 smithi179 ceph-mon[25623]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:25.136 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:24 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:25.136 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:24 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:25.137 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:24.785+0000 7faa45948200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:05:25.137 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:24.878+0000 7faa45948200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:05:25.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:24 smithi175 ceph-mon[24776]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:25.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:24 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:25.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:24 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:25.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:05:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:24] "GET /metrics HTTP/1.1" 200 208030 "" "Prometheus/2.33.4" 2024-05-25T06:05:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:24 smithi175 ceph-mon[29436]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:24 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:24 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:05:25.494 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:25.135+0000 7faa45948200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:05:25.494 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:25.275+0000 7faa45948200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:05:25.494 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:25.374+0000 7faa45948200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:05:25.494 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:25.493+0000 7faa45948200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:05:26.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:25.801+0000 7faa45948200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:26.410+0000 7faa45948200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:26.630+0000 7faa45948200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:05:26] ENGINE Bus STARTING 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: CherryPy Checker: 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: The Application mounted at '' has an empty config. 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:05:26] ENGINE Serving on http://:::9283 2024-05-25T06:05:26.739 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:05:26] ENGINE Bus STARTED 2024-05-25T06:05:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:26 smithi179 ceph-mon[25623]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:26 smithi179 ceph-mon[25623]: Standby manager daemon x restarted 2024-05-25T06:05:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:26 smithi179 ceph-mon[25623]: Standby manager daemon x started 2024-05-25T06:05:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:26 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:05:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:26 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:05:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:26 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:05:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:26 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[24776]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[24776]: Standby manager daemon x restarted 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[24776]: Standby manager daemon x started 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[29436]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[29436]: Standby manager daemon x restarted 2024-05-25T06:05:27.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T06:05:27.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:05:27.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:05:27.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:05:27.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:26 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:05:28.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:27 smithi179 ceph-mon[25623]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-25T06:05:28.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:27 smithi175 ceph-mon[24776]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-25T06:05:28.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:27 smithi175 ceph-mon[29436]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-25T06:05:28.649 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:28.348Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:29.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:28 smithi179 ceph-mon[25623]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:29.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:28 smithi175 ceph-mon[24776]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:29.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:28 smithi175 ceph-mon[29436]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:30.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:30 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:30.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:30 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:30.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:30 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:31 smithi179 ceph-mon[25623]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:31.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:31 smithi175 ceph-mon[24776]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:31 smithi175 ceph-mon[29436]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:33 smithi179 ceph-mon[25623]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:33.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:33 smithi175 ceph-mon[24776]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:33.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:33 smithi175 ceph-mon[29436]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:05:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:35 smithi179 ceph-mon[25623]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:35.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:35 smithi175 ceph-mon[24776]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:35.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:35 smithi175 ceph-mon[29436]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:35.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:05:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:34] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-05-25T06:05:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:05:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:37 smithi179 ceph-mon[25623]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:37.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:37 smithi175 ceph-mon[24776]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:37.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:37 smithi175 ceph-mon[29436]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:39 smithi179 ceph-mon[25623]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:39 smithi175 ceph-mon[24776]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:39.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:39 smithi175 ceph-mon[29436]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:40.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:40 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:40.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:40 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:40.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:40 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:41.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:41 smithi179 ceph-mon[25623]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:41.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:41 smithi175 ceph-mon[24776]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:41.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:41 smithi175 ceph-mon[29436]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:43.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:43 smithi179 ceph-mon[25623]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:43.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:43 smithi175 ceph-mon[24776]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:43 smithi175 ceph-mon[29436]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:43.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:05:43.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:43.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:43.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:05:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:05:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:05:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:05:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:05:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:05:45.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:45 smithi179 ceph-mon[25623]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:45.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:45 smithi175 ceph-mon[29436]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:45.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:45 smithi175 ceph-mon[24776]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:45.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:05:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:44] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-05-25T06:05:45.767 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:05:47.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:47 smithi175 ceph-mon[29436]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:47.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:47 smithi175 ceph-mon[24776]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:47.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:47 smithi179 ceph-mon[25623]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:49.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:49 smithi175 ceph-mon[24776]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:49.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:49 smithi175 ceph-mon[29436]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:49 smithi179 ceph-mon[25623]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:50 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:50.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:50 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:50.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:50 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:05:51.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:51 smithi175 ceph-mon[24776]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:51.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:51 smithi175 ceph-mon[29436]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:51 smithi179 ceph-mon[25623]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:53.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:53 smithi175 ceph-mon[24776]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:53.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:53 smithi175 ceph-mon[29436]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:53.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:53 smithi179 ceph-mon[25623]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:05:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:05:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:05:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:05:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:05:55.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:55 smithi179 ceph-mon[25623]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:55.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:55 smithi175 ceph-mon[24776]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:55.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:55 smithi175 ceph-mon[29436]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:55.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:05:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:54] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-05-25T06:05:55.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:05:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:05:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:05:57.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:57 smithi175 ceph-mon[24776]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:57.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:57 smithi175 ceph-mon[29436]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:57.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:57 smithi179 ceph-mon[25623]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:05:59.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:05:59 smithi175 ceph-mon[24776]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:59.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:05:59 smithi175 ceph-mon[29436]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:05:59.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:05:59 smithi179 ceph-mon[25623]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:00.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:00 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:00.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:00 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:00.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:00 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:01 smithi179 ceph-mon[25623]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:01.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:01 smithi175 ceph-mon[24776]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:01.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:01 smithi175 ceph-mon[29436]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:03.510 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:03 smithi175 ceph-mon[24776]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:03.511 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:03 smithi175 ceph-mon[29436]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:03.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:03 smithi179 ceph-mon[25623]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:03.819 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:06:03.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:05.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:05 smithi175 ceph-mon[24776]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:05.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:06:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:04] "GET /metrics HTTP/1.1" 200 208028 "" "Prometheus/2.33.4" 2024-05-25T06:06:05.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:05 smithi175 ceph-mon[29436]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:05.337 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:05 smithi179 ceph-mon[25623]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:05.337 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:06:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:06:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:07 smithi179 ceph-mon[25623]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:07.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:07 smithi175 ceph-mon[24776]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:07.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:07 smithi175 ceph-mon[29436]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:09.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:09 smithi179 ceph-mon[25623]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:09.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:09 smithi175 ceph-mon[24776]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:09.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:09 smithi175 ceph-mon[29436]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:10 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:10.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:10 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:10.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:10 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:11 smithi179 ceph-mon[25623]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:11.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:11 smithi175 ceph-mon[24776]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:11.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:11 smithi175 ceph-mon[29436]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:13.512 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:13 smithi175 ceph-mon[29436]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:13.512 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:13 smithi175 ceph-mon[24776]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:13 smithi179 ceph-mon[25623]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:13.819 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:06:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:15.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:15 smithi175 ceph-mon[24776]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:15.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:06:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:14] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-05-25T06:06:15.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:15 smithi175 ceph-mon[29436]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:15.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:15 smithi179 ceph-mon[25623]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:15.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:06:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:06:17.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:17 smithi179 ceph-mon[25623]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:17.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:17 smithi175 ceph-mon[24776]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:17 smithi175 ceph-mon[29436]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:19 smithi179 ceph-mon[25623]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:19.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:19 smithi175 ceph-mon[24776]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:19 smithi175 ceph-mon[29436]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:20 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:20.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:20 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:20 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:21 smithi179 ceph-mon[25623]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:21.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:21 smithi175 ceph-mon[24776]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:21.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:21 smithi175 ceph-mon[29436]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:23.512 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:23 smithi175 ceph-mon[24776]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:23.512 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:23 smithi175 ceph-mon[29436]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:23.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:23 smithi179 ceph-mon[25623]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:06:23.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:25.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:06:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:24] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-05-25T06:06:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[29436]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:06:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:06:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:06:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:06:25.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[24776]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:25.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:06:25.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:06:25.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:06:25.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:25 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:06:25.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:06:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:06:25.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:25 smithi179 ceph-mon[25623]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:25.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:25 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:06:25.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:25 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:06:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:25 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:06:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:25 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' 2024-05-25T06:06:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:27 smithi179 ceph-mon[25623]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:27.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:27 smithi175 ceph-mon[29436]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:27.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:27 smithi175 ceph-mon[24776]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:29 smithi179 ceph-mon[25623]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:29.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:29 smithi175 ceph-mon[24776]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:29 smithi175 ceph-mon[29436]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:30 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:30.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:30 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:30 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:31 smithi179 ceph-mon[25623]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:31.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:31 smithi175 ceph-mon[24776]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:31 smithi175 ceph-mon[29436]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:33.512 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:33 smithi175 ceph-mon[29436]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:33.513 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:33 smithi175 ceph-mon[24776]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:33 smithi179 ceph-mon[25623]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:06:33.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:35 smithi175 ceph-mon[24776]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:35.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:35 smithi175 ceph-mon[29436]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:35.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:06:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:34] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-05-25T06:06:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:35 smithi179 ceph-mon[25623]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:35.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:06:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:06:37.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:37 smithi179 ceph-mon[25623]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:37.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:37 smithi175 ceph-mon[24776]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:37.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:37 smithi175 ceph-mon[29436]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:39.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:39 smithi179 ceph-mon[25623]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:39.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:39 smithi175 ceph-mon[24776]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:39.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:39 smithi175 ceph-mon[29436]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:40.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:40 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:40.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:40 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:40 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:41.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:41 smithi179 ceph-mon[25623]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:41.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:41 smithi175 ceph-mon[24776]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:41.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:41 smithi175 ceph-mon[29436]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:43.514 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:43 smithi175 ceph-mon[24776]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:43.514 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:43 smithi175 ceph-mon[29436]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:43 smithi179 ceph-mon[25623]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:43.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:06:43.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:43.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:43.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:06:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:44 smithi179 ceph-mon[25623]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:06:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:06:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:44 smithi179 ceph-mon[25623]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[24776]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[24776]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[29436]: from='mgr.24340 172.21.15.175:0/1104111271' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:06:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:44 smithi175 ceph-mon[29436]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:06:45.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:45 smithi175 ceph-mon[24776]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:45.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:45 smithi175 ceph-mon[29436]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:45.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:06:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:44] "GET /metrics HTTP/1.1" 200 208036 "" "Prometheus/2.33.4" 2024-05-25T06:06:45.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:45 smithi179 ceph-mon[25623]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:45.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:06:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:06:47.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:47 smithi179 ceph-mon[25623]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:47 smithi175 ceph-mon[24776]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:47.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:47 smithi175 ceph-mon[29436]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:49 smithi179 ceph-mon[25623]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:49.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:49 smithi175 ceph-mon[24776]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:49 smithi175 ceph-mon[29436]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:50.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:50 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:50.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:50 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:50 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:51 smithi179 ceph-mon[25623]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:51.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:51 smithi175 ceph-mon[29436]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:51 smithi175 ceph-mon[24776]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:53.514 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:53 smithi175 ceph-mon[29436]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:53.515 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:53 smithi175 ceph-mon[24776]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:53.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:53 smithi179 ceph-mon[25623]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:53.819 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:06:53.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:06:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:06:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:06:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:06:55.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:06:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:54] "GET /metrics HTTP/1.1" 200 208036 "" "Prometheus/2.33.4" 2024-05-25T06:06:55.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:55 smithi179 ceph-mon[25623]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:55.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:06:55 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:06:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:06:55.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:55 smithi175 ceph-mon[24776]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:55.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:55 smithi175 ceph-mon[29436]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:57.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:57 smithi179 ceph-mon[25623]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:57.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:57 smithi175 ceph-mon[24776]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:57.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:57 smithi175 ceph-mon[29436]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:06:58.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:58 smithi179 ceph-mon[25623]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:58.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:58 smithi175 ceph-mon[24776]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:58.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:58 smithi175 ceph-mon[29436]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:06:59.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:06:59 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:59.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:06:59 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:06:59.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:06:59 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:00 smithi179 ceph-mon[25623]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:00.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:00 smithi175 ceph-mon[24776]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:00 smithi175 ceph-mon[29436]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:02 smithi179 ceph-mon[25623]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:02.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:02 smithi175 ceph-mon[24776]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:02 smithi175 ceph-mon[29436]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:03.819 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:07:03.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:07:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:07:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:07:04.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:04 smithi179 ceph-mon[25623]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:04.810 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:04 smithi175 ceph-mon[24776]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:04.810 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:04 smithi175 ceph-mon[29436]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:05.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:04] "GET /metrics HTTP/1.1" 200 208030 "" "Prometheus/2.33.4" 2024-05-25T06:07:05.474 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:07:05.686 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:07:06.764 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:06 smithi175 ceph-mon[29436]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:06.765 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:06 smithi175 ceph-mon[24776]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:06 smithi179 ceph-mon[25623]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:07.370 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:07:07.370 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (6m) 2m ago 7m 26.2M - ba2b418f427c aa07cecebf6f 2024-05-25T06:07:07.370 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (6m) 103s ago 6m 54.1M - 8.3.5 dad864ee21e9 c3a0d6b80f61 2024-05-25T06:07:07.370 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (6m) 2m ago 6m 49.1M - 3.5 e1d6a67b021e cbf3ec05b70e 2024-05-25T06:07:07.370 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283 running (108s) 103s ago 11m 65.8M - 19.0.0-2843-gea0684c0 f9a9809cafd6 2c455cbdf457 2024-05-25T06:07:07.370 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:9283 running (13m) 2m ago 13m 461M - 17.2.0 e1d6a67b021e 8822d7e4e7de 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (13m) 2m ago 13m 46.9M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (12m) 103s ago 12m 40.2M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (12m) 2m ago 12m 41.4M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (8m) 2m ago 8m 18.0M - 1dbe0e931976 cbca05d03edb 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (7m) 103s ago 7m 17.4M - 1dbe0e931976 31b85a1d8b12 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (11m) 2m ago 11m 52.5M 4016M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (11m) 2m ago 11m 53.8M 4016M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:07:07.371 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (10m) 2m ago 10m 48.7M 4016M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:07:07.372 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (10m) 2m ago 10m 48.9M 4016M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:07:07.372 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (9m) 103s ago 9m 49.4M 4272M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:07:07.372 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (9m) 103s ago 9m 48.9M 4272M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:07:07.372 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (9m) 103s ago 9m 49.9M 4272M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:07:07.372 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (8m) 103s ago 8m 51.4M 4272M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:07:07.372 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (6m) 103s ago 8m 67.7M - 514e6a882f6e 8c1fe73f54ff 2024-05-25T06:07:07.372 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (6m) 2m ago 6m 70.5M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:07:07.373 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (6m) 103s ago 6m 74.0M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:07:08.011 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions' 2024-05-25T06:07:08.550 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:08 smithi175 ceph-mon[29436]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:08.550 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:08 smithi175 ceph-mon[29436]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:07:08.550 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:08 smithi175 ceph-mon[24776]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:08.550 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:08 smithi175 ceph-mon[24776]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:07:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:08 smithi179 ceph-mon[25623]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:08 smithi179 ceph-mon[25623]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:07:09.648 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:09 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:09.649 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:09 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:09 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:09.880 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 1 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:07:09.881 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 1 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:07:09.882 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:07:10.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph -s' 2024-05-25T06:07:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:10 smithi179 ceph-mon[25623]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:10 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1436692426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:07:10.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:10 smithi175 ceph-mon[29436]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:10 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1436692426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:07:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:10 smithi175 ceph-mon[24776]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:10 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1436692426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:07:12.525 INFO:teuthology.orchestra.run.smithi175.stdout: cluster: 2024-05-25T06:07:12.525 INFO:teuthology.orchestra.run.smithi175.stdout: id: 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:07:12.525 INFO:teuthology.orchestra.run.smithi175.stdout: health: HEALTH_OK 2024-05-25T06:07:12.525 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:07:12.525 INFO:teuthology.orchestra.run.smithi175.stdout: services: 2024-05-25T06:07:12.525 INFO:teuthology.orchestra.run.smithi175.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-05-25T06:07:12.525 INFO:teuthology.orchestra.run.smithi175.stdout: mgr: y(active, since 8m), standbys: x 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: data: 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: pools: 6 pools, 161 pgs 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: objects: 241 objects, 585 KiB 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: pgs: 161 active+clean 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:07:12.526 INFO:teuthology.orchestra.run.smithi175.stdout: io: 2024-05-25T06:07:12.527 INFO:teuthology.orchestra.run.smithi175.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-25T06:07:12.527 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:07:12.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:12 smithi179 ceph-mon[25623]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:12.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:12 smithi175 ceph-mon[29436]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:12 smithi175 ceph-mon[24776]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:13.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:07:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:13 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2808100568' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:07:13.799 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:13 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2808100568' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:07:13.800 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:13 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2808100568' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:07:13.800 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:07:13.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:07:13.800 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:07:13.800 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:07:14.698 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:14 smithi175 ceph-mon[29436]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:14.699 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:14 smithi175 ceph-mon[24776]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:14 smithi179 ceph-mon[25623]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:15.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:14] "GET /metrics HTTP/1.1" 200 208037 "" "Prometheus/2.33.4" 2024-05-25T06:07:15.167 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_OK 2024-05-25T06:07:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:15 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2552021735' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:07:15.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:07:15.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:15 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2552021735' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:07:15.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:15 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2552021735' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:07:15.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-05-25T06:07:16.785 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:16 smithi175 ceph-mon[29436]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:16.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:16 smithi175 ceph-mon[24776]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:16 smithi179 ceph-mon[25623]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:17.717 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:07:18.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph mgr fail' 2024-05-25T06:07:18.558 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:18 smithi175 ceph-mon[24776]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:18.558 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:18 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1938494791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:07:18.558 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:18 smithi175 ceph-mon[29436]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:18.558 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:18 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1938494791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:07:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:18 smithi179 ceph-mon[25623]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:18 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1938494791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:07:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:19 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:19.801 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:19 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:19.801 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:19 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:20 smithi179 ceph-mon[25623]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:20 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1390533776' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:07:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:20 smithi179 ceph-mon[25623]: osdmap e97: 8 total, 8 up, 8 in 2024-05-25T06:07:20.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:20] ENGINE Bus STOPPING 2024-05-25T06:07:20.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:07:20.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:20] ENGINE Bus STOPPED 2024-05-25T06:07:20.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:20] ENGINE Bus STARTING 2024-05-25T06:07:20.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:20] ENGINE Serving on http://:::9283 2024-05-25T06:07:20.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:20] ENGINE Bus STARTED 2024-05-25T06:07:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:20 smithi175 ceph-mon[29436]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:20 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1390533776' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:07:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:20 smithi175 ceph-mon[29436]: osdmap e97: 8 total, 8 up, 8 in 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:20.520+0000 7f055e53b700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setuser ceph since I am not root 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ignoring --setgroup ceph since I am not root 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:20 smithi175 ceph-mgr[25008]: -- 172.21.15.175:0/948781265 <== mon.0 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5602539e8340 con 0x5602539c0800 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:20.743+0000 7f45ba1d1000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:20 smithi175 ceph-mon[24776]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:20 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1390533776' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:07:20.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:20 smithi175 ceph-mon[24776]: osdmap e97: 8 total, 8 up, 8 in 2024-05-25T06:07:21.231 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:20 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:20.889+0000 7f45ba1d1000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:07:21.231 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:21.001+0000 7f45ba1d1000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:07:21.258 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'sleep 180' 2024-05-25T06:07:21.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:21.338+0000 7f45ba1d1000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:07:21.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:21.425+0000 7f45ba1d1000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:07:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1390533776' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: mgrmap e22: x(active, starting, since 0.392557s) 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:07:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: Manager daemon x is now available 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: Queued rgw.foo for migration 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:07:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:07:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:07:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:07:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: Deploying cephadm binary to smithi179 2024-05-25T06:07:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: Deploying cephadm binary to smithi175 2024-05-25T06:07:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:21 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1390533776' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: mgrmap e22: x(active, starting, since 0.392557s) 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:07:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: Manager daemon x is now available 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: Queued rgw.foo for migration 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:07:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: Deploying cephadm binary to smithi179 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: Deploying cephadm binary to smithi175 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1390533776' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:07:21.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: mgrmap e22: x(active, starting, since 0.392557s) 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:07:21.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:07:21.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:07:21.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:07:21.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:07:21.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:07:21.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:07:21.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: Manager daemon x is now available 2024-05-25T06:07:21.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: Queued rgw.foo for migration 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: Deploying cephadm binary to smithi179 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: Deploying cephadm binary to smithi175 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:21.826 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:21 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:22.773 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:22.609+0000 7f45ba1d1000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:07:22.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[29436]: mgrmap e23: x(active, since 1.40205s) 2024-05-25T06:07:22.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[29436]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:22.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[29436]: [25/May/2024:06:07:21] ENGINE Bus STARTING 2024-05-25T06:07:22.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[29436]: [25/May/2024:06:07:21] ENGINE Serving on https://172.21.15.179:7150 2024-05-25T06:07:22.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[29436]: [25/May/2024:06:07:21] ENGINE Serving on http://172.21.15.179:8765 2024-05-25T06:07:22.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[29436]: [25/May/2024:06:07:21] ENGINE Bus STARTED 2024-05-25T06:07:22.774 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[24776]: mgrmap e23: x(active, since 1.40205s) 2024-05-25T06:07:22.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[24776]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:22.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[24776]: [25/May/2024:06:07:21] ENGINE Bus STARTING 2024-05-25T06:07:22.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[24776]: [25/May/2024:06:07:21] ENGINE Serving on https://172.21.15.179:7150 2024-05-25T06:07:22.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[24776]: [25/May/2024:06:07:21] ENGINE Serving on http://172.21.15.179:8765 2024-05-25T06:07:22.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:22 smithi175 ceph-mon[24776]: [25/May/2024:06:07:21] ENGINE Bus STARTED 2024-05-25T06:07:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:22 smithi179 ceph-mon[25623]: mgrmap e23: x(active, since 1.40205s) 2024-05-25T06:07:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:22 smithi179 ceph-mon[25623]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:22 smithi179 ceph-mon[25623]: [25/May/2024:06:07:21] ENGINE Bus STARTING 2024-05-25T06:07:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:22 smithi179 ceph-mon[25623]: [25/May/2024:06:07:21] ENGINE Serving on https://172.21.15.179:7150 2024-05-25T06:07:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:22 smithi179 ceph-mon[25623]: [25/May/2024:06:07:21] ENGINE Serving on http://172.21.15.179:8765 2024-05-25T06:07:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:22 smithi179 ceph-mon[25623]: [25/May/2024:06:07:21] ENGINE Bus STARTED 2024-05-25T06:07:23.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:22.781+0000 7f45ba1d1000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:07:23.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:22.865+0000 7f45ba1d1000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:07:23.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:23.185+0000 7f45ba1d1000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:07:23.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:23.278+0000 7f45ba1d1000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:07:23.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:23.372+0000 7f45ba1d1000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:07:23.517 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:23.334Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:07:23.517 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:07:23.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:07:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:23 smithi179 ceph-mon[25623]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:23.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:23 smithi175 ceph-mon[29436]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:23 smithi175 ceph-mon[24776]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:23.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:23.620+0000 7f45ba1d1000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:07:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:07:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:23.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:07:24.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:24.063+0000 7f45ba1d1000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:07:24.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:24.553+0000 7f45ba1d1000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:07:24.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:24.648+0000 7f45ba1d1000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:07:25.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:24.945+0000 7f45ba1d1000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:07:25.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:25.036+0000 7f45ba1d1000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:07:25.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:25] "GET /metrics HTTP/1.1" 200 34907 "" "Prometheus/2.33.4" 2024-05-25T06:07:25.793 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:25.445+0000 7f45ba1d1000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:07:25.793 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:25.581+0000 7f45ba1d1000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:07:25.793 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:25.676+0000 7f45ba1d1000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:07:25.793 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:25.793+0000 7f45ba1d1000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:07:26.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:25.891+0000 7f45ba1d1000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:07:26.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:26 smithi179 ceph-mon[25623]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:26.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:26 smithi179 ceph-mon[25623]: mgrmap e24: x(active, since 5s) 2024-05-25T06:07:26.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:26.489+0000 7f45ba1d1000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:07:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:26 smithi175 ceph-mon[24776]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:26 smithi175 ceph-mon[24776]: mgrmap e24: x(active, since 5s) 2024-05-25T06:07:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:26 smithi175 ceph-mon[29436]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:26 smithi175 ceph-mon[29436]: mgrmap e24: x(active, since 5s) 2024-05-25T06:07:27.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:26.865+0000 7f45ba1d1000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:07:27.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:06:07:26] ENGINE Bus STARTING 2024-05-25T06:07:27.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: CherryPy Checker: 2024-05-25T06:07:27.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: The Application mounted at '' has an empty config. 2024-05-25T06:07:27.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: 2024-05-25T06:07:27.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:06:07:26] ENGINE Serving on http://:::9283 2024-05-25T06:07:27.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:26 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: [25/May/2024:06:07:26] ENGINE Bus STARTED 2024-05-25T06:07:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:27 smithi179 ceph-mon[25623]: Standby manager daemon y started 2024-05-25T06:07:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:27 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:07:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:27 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:07:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:27 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:07:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:27 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:07:27.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:27 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:27.585Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:07:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[24776]: Standby manager daemon y started 2024-05-25T06:07:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:07:27.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:07:27.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:07:27.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:07:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[29436]: Standby manager daemon y started 2024-05-25T06:07:27.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:07:27.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:07:27.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:07:27.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:27 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2359776090' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:07:28.735 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:28 smithi175 ceph-mon[29436]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:28.735 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:28 smithi175 ceph-mon[29436]: mgrmap e25: x(active, since 7s), standbys: y 2024-05-25T06:07:28.735 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:28 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:07:28.735 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:28 smithi175 ceph-mon[24776]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:28.735 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:28 smithi175 ceph-mon[24776]: mgrmap e25: x(active, since 7s), standbys: y 2024-05-25T06:07:28.735 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:28 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:07:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:28 smithi179 ceph-mon[25623]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:28 smithi179 ceph-mon[25623]: mgrmap e25: x(active, since 7s), standbys: y 2024-05-25T06:07:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:28 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:07:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:30 smithi179 ceph-mon[25623]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:07:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:30 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:30 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:30 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:30 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:30 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[29436]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[24776]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:30 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi175 to 2992M 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:31.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:31 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi175 to 2992M 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi175 to 2992M 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-25T06:07:31.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:31.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:31.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:31 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:07:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:07:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: Adjusting osd_memory_target on smithi179 to 3248M 2024-05-25T06:07:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:32 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: Adjusting osd_memory_target on smithi179 to 3248M 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: Adjusting osd_memory_target on smithi179 to 3248M 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:32 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:32 smithi175 systemd[1]: Stopping Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:33.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:32 smithi175 bash[58248]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter.a" found: no such container 2024-05-25T06:07:33.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:32 smithi175 podman[58260]: 2024-05-25 06:07:32.969602418 +0000 UTC m=+0.138182925 container died cbca05d03edbb888a98756eff7a41eb2fa860722666a537e2a47f21e2e6e36e1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:33.520 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=error ts=2024-05-25T06:07:33.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:07:33.520 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 podman[58260]: 2024-05-25 06:07:33.350986109 +0000 UTC m=+0.519566603 container cleanup cbca05d03edbb888a98756eff7a41eb2fa860722666a537e2a47f21e2e6e36e1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:33.520 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 bash[58260]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a 2024-05-25T06:07:33.520 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 bash[58286]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter.a" found: no such container 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-25T06:07:33.521 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[29436]: Reconfiguring daemon node-exporter.a on smithi175 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-25T06:07:33.522 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-mon[24776]: Reconfiguring daemon node-exporter.a on smithi175 2024-05-25T06:07:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:07:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:07:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:07:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:07:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:33.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-25T06:07:33.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:33 smithi179 ceph-mon[25623]: Reconfiguring daemon node-exporter.a on smithi175 2024-05-25T06:07:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.175:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.175 because it doesn't contain any IP SANs" 2024-05-25T06:07:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=warn ts=2024-05-25T06:07:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.179:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.179 because it doesn't contain any IP SANs" 2024-05-25T06:07:33.820 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 podman[58275]: 2024-05-25 06:07:33.618821671 +0000 UTC m=+0.652659910 container remove cbca05d03edbb888a98756eff7a41eb2fa860722666a537e2a47f21e2e6e36e1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:33.820 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:07:33.820 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-25T06:07:33.820 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 systemd[1]: Stopped Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:33.820 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Consumed 3.772s CPU time. 2024-05-25T06:07:33.820 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 systemd[1]: Starting Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:34.103 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 podman[58350]: 2024-05-25 06:07:33.755350962 +0000 UTC m=+0.016690198 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-25T06:07:34.103 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:33 smithi175 podman[58350]: 2024-05-25 06:07:33.87838256 +0000 UTC m=+0.139721791 container create 3fd7592e3fcc03dee13fd956ce62d457fab9c29222ffc8ebd88d44cba7bce868 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:34.103 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 podman[58350]: 2024-05-25 06:07:34.097337415 +0000 UTC m=+0.358676651 container init 3fd7592e3fcc03dee13fd956ce62d457fab9c29222ffc8ebd88d44cba7bce868 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:34.103 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 podman[58350]: 2024-05-25 06:07:34.100096927 +0000 UTC m=+0.361436166 container start 3fd7592e3fcc03dee13fd956ce62d457fab9c29222ffc8ebd88d44cba7bce868 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:34.103 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.103Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-25T06:07:34.103 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.103Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-25T06:07:34.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:34 smithi175 ceph-mon[29436]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-25T06:07:34.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:34 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:34.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:34 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-mon[24776]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-25T06:07:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=arp 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-25T06:07:34.571 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=edac 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-25T06:07:34.572 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=os 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=stat 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-25T06:07:34.573 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=time 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=uname 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.104Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.105Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.105Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[58365]: ts=2024-05-25T06:07:34.105Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 bash[58350]: 3fd7592e3fcc03dee13fd956ce62d457fab9c29222ffc8ebd88d44cba7bce868 2024-05-25T06:07:34.574 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:07:34 smithi175 systemd[1]: Started Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:34 smithi179 ceph-mon[25623]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-25T06:07:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:34 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:34 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:35.146 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:07:35.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:35 smithi179 ceph-mon[25623]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-25T06:07:35.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:35 smithi179 ceph-mon[25623]: Reconfiguring daemon alertmanager.a on smithi175 2024-05-25T06:07:35.687 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:35] "GET /metrics HTTP/1.1" 200 34907 "" "Prometheus/2.33.4" 2024-05-25T06:07:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:35 smithi175 ceph-mon[29436]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-25T06:07:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:35 smithi175 ceph-mon[29436]: Reconfiguring daemon alertmanager.a on smithi175 2024-05-25T06:07:35.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:35 smithi175 ceph-mon[24776]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-25T06:07:35.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:35 smithi175 ceph-mon[24776]: Reconfiguring daemon alertmanager.a on smithi175 2024-05-25T06:07:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:36 smithi179 ceph-mon[25623]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:36 smithi175 ceph-mon[24776]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:36 smithi175 ceph-mon[29436]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:37.408 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:37 smithi175 systemd[1]: Stopping Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:37.409 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:37 smithi175 bash[58537]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager.a" found: no such container 2024-05-25T06:07:37.717 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[49343]: level=info ts=2024-05-25T06:07:37.408Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:07:37.717 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:37 smithi175 podman[58549]: 2024-05-25 06:07:37.429611127 +0000 UTC m=+0.132305107 container died aa07cecebf6fcae15d93952c9cbca8b9022d5418b17bcfa12303e359f9004763 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:38.059 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:37 smithi175 podman[58549]: 2024-05-25 06:07:37.825565845 +0000 UTC m=+0.528259826 container cleanup aa07cecebf6fcae15d93952c9cbca8b9022d5418b17bcfa12303e359f9004763 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:38.059 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:37 smithi175 bash[58549]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a 2024-05-25T06:07:38.059 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:37 smithi175 bash[58574]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager.a" found: no such container 2024-05-25T06:07:38.319 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 podman[58561]: 2024-05-25 06:07:38.060078385 +0000 UTC m=+0.629337712 container remove aa07cecebf6fcae15d93952c9cbca8b9022d5418b17bcfa12303e359f9004763 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:38.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 podman[58561]: 2024-05-25 06:07:38.144411599 +0000 UTC m=+0.713670881 volume remove 47294d149029b72d85f1b9abd1028d661e3d54d86fddf890ddc1e63927e8465e 2024-05-25T06:07:38.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-25T06:07:38.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 systemd[1]: Stopped Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:38.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service: Consumed 1.054s CPU time. 2024-05-25T06:07:38.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 systemd[1]: Starting Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:38.634 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-mon[24776]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:38.634 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:38 smithi175 ceph-mon[29436]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:38.635 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 podman[58637]: 2024-05-25 06:07:38.280170163 +0000 UTC m=+0.017922720 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-25T06:07:38.635 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 podman[58637]: 2024-05-25 06:07:38.386977867 +0000 UTC m=+0.124730423 volume create 868acd48a0a36d04a77ffa65ba7e620368272fdc7c578262682c1f9a5d69864d 2024-05-25T06:07:38.635 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 podman[58637]: 2024-05-25 06:07:38.45397518 +0000 UTC m=+0.191727779 container create 1893f9da103a12c45b7bda5555f746a88f977f422999d56d7ff5e568e23b0635 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:38 smithi179 ceph-mon[25623]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:39.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 podman[58637]: 2024-05-25 06:07:38.723539965 +0000 UTC m=+0.461292531 container init 1893f9da103a12c45b7bda5555f746a88f977f422999d56d7ff5e568e23b0635 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:39.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 podman[58637]: 2024-05-25 06:07:38.726500278 +0000 UTC m=+0.464252840 container start 1893f9da103a12c45b7bda5555f746a88f977f422999d56d7ff5e568e23b0635 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:39.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.736Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-25T06:07:39.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.736Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-25T06:07:39.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.737Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.175 port=9094 2024-05-25T06:07:39.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.738Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-25T06:07:39.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.763Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T06:07:39.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.763Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T06:07:39.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.765Z caller=main.go:518 msg=Listening address=:9093 2024-05-25T06:07:39.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:38.765Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-25T06:07:39.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 bash[58637]: 1893f9da103a12c45b7bda5555f746a88f977f422999d56d7ff5e568e23b0635 2024-05-25T06:07:39.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:38 smithi175 systemd[1]: Started Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:07:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:07:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:40.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:40.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:40.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:39 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:07:40.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:07:40.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: from='client.24694 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:07:40.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:40.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:40.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:07:40.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:40.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:40.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:40.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:39 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:07:41.016 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:40.739Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000397045s 2024-05-25T06:07:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:41 smithi179 ceph-mon[25623]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:42.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:07:41.992+0000 7fa9cb44e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:42.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:07:42.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:42.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:42.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:42.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:42.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:42.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:42.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:42.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:07:42.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:07:42.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:07:42.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return future.result(timeout) 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.__get_result() 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: raise self._exception 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: use_current_daemon_image=reconfig, 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:42.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:42.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:42.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:42.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:42.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:42.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:42.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:42.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:41 smithi175 ceph-mon[24776]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:42.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:41 smithi175 ceph-mon[29436]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:42.783 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:42 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.179:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.175:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:07:43.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:42 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:07:43.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:43.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.041 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:42 smithi179 ceph-mon[25623]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:07:43.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:42 smithi179 ceph-mon[25623]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:42 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:07:43.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:43.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:42 smithi175 ceph-mon[24776]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:07:43.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:42 smithi175 ceph-mon[24776]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:42 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:07:43.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:07:43.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:42 smithi175 ceph-mon[29436]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:07:43.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:42 smithi175 ceph-mon[29436]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:07:44.177 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:43 smithi179 ceph-mon[25623]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:43 smithi175 ceph-mon[24776]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:43 smithi175 ceph-mon[29436]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:07:45.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:07:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[25004]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:07:45.471 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:45] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-25T06:07:45.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 systemd[1]: Stopping Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:45.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 bash[48688]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.785Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.785Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.785Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.785Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.785Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.785Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.786Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.786Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.786Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-25T06:07:46.062 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.786Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-25T06:07:46.063 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.786Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-25T06:07:46.063 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[42393]: ts=2024-05-25T06:07:45.786Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-25T06:07:46.063 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:45 smithi179 podman[48699]: 2024-05-25 06:07:45.796020316 +0000 UTC m=+0.129617758 container died 8c1fe73f54ff905cdd88c8f2b62ca2e2e719852d54ff68793c0c2495a4f4f191 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:46.402 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 podman[48699]: 2024-05-25 06:07:46.151750941 +0000 UTC m=+0.485348370 container cleanup 8c1fe73f54ff905cdd88c8f2b62ca2e2e719852d54ff68793c0c2495a4f4f191 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:46.402 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 bash[48699]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a 2024-05-25T06:07:46.403 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 bash[48726]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:07:46.678 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:46 smithi179 ceph-mon[25623]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:46.678 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 podman[48712]: 2024-05-25 06:07:46.403263814 +0000 UTC m=+0.608928658 container remove 8c1fe73f54ff905cdd88c8f2b62ca2e2e719852d54ff68793c0c2495a4f4f191 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:46.679 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-25T06:07:46.679 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 systemd[1]: Stopped Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:46.679 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Consumed 1.957s CPU time. 2024-05-25T06:07:46.679 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 systemd[1]: Starting Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:46.679 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 podman[48787]: 2024-05-25 06:07:46.539266658 +0000 UTC m=+0.018471158 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-25T06:07:46.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:46 smithi175 ceph-mon[24776]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:46 smithi175 ceph-mon[29436]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:46.964 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 podman[48787]: 2024-05-25 06:07:46.679333658 +0000 UTC m=+0.158538138 container create b84f89714aa431a779540344d69987c8d33167e13db3d9f797bdbb2aea97d7cc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:46.964 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 podman[48787]: 2024-05-25 06:07:46.88995705 +0000 UTC m=+0.369161552 container init b84f89714aa431a779540344d69987c8d33167e13db3d9f797bdbb2aea97d7cc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:46.964 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 podman[48787]: 2024-05-25 06:07:46.892762682 +0000 UTC m=+0.371967176 container start b84f89714aa431a779540344d69987c8d33167e13db3d9f797bdbb2aea97d7cc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.916Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.916Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.916Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.916Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi179 (none))" 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.916Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.916Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.917Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.918Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.918Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.920Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.921Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=678.379µs 2024-05-25T06:07:46.965 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:46.921Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-25T06:07:46.966 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 bash[48787]: b84f89714aa431a779540344d69987c8d33167e13db3d9f797bdbb2aea97d7cc 2024-05-25T06:07:47.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:46 smithi179 systemd[1]: Started Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:47.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.129Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-05-25T06:07:47.728 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:47 smithi179 systemd[1]: Stopping Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:47.728 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:47 smithi179 bash[48910]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter.b" found: no such container 2024-05-25T06:07:47.728 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.360Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-05-25T06:07:47.728 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.360Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-05-25T06:07:47.728 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.360Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=52.337µs wal_replay_duration=439.017548ms total_replay_duration=439.76563ms 2024-05-25T06:07:47.728 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.363Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-25T06:07:47.728 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.363Z caller=main.go:947 level=info msg="TSDB started" 2024-05-25T06:07:47.728 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.363Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-25T06:07:47.729 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.386Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.972588ms db_storage=752ns remote_storage=1.226µs web_handler=308ns query_engine=879ns scrape=426.45µs scrape_sd=96.332µs notify=14.594µs notify_sd=29.861µs rules=21.999896ms 2024-05-25T06:07:47.729 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:47.386Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-25T06:07:47.996 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:47 smithi179 podman[48921]: 2024-05-25 06:07:47.728794932 +0000 UTC m=+0.140475663 container died 31b85a1d8b12426ada30191ebda46a839ab0b91a17f3fafb0b30f7bdaa99fb42 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:07:48.269 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 podman[48921]: 2024-05-25 06:07:48.135097678 +0000 UTC m=+0.546778430 container cleanup 31b85a1d8b12426ada30191ebda46a839ab0b91a17f3fafb0b30f7bdaa99fb42 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:07:48.269 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 bash[48921]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b 2024-05-25T06:07:48.269 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:47 smithi179 ceph-mon[25623]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:48.269 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:47 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:48.269 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:47 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:48.269 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:47 smithi179 ceph-mon[25623]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-25T06:07:48.269 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:47 smithi179 ceph-mon[25623]: Reconfiguring daemon node-exporter.b on smithi179 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[24776]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[24776]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[24776]: Reconfiguring daemon node-exporter.b on smithi179 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[29436]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:48.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:48.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[29436]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-25T06:07:48.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:47 smithi175 ceph-mon[29436]: Reconfiguring daemon node-exporter.b on smithi179 2024-05-25T06:07:48.522 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 podman[48935]: 2024-05-25 06:07:48.26956702 +0000 UTC m=+0.544708768 container remove 31b85a1d8b12426ada30191ebda46a839ab0b91a17f3fafb0b30f7bdaa99fb42 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:07:48.523 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:07:48.523 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 bash[48949]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter.b" found: no such container 2024-05-25T06:07:48.523 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-25T06:07:48.523 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 systemd[1]: Stopped Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:48.523 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Consumed 3.795s CPU time. 2024-05-25T06:07:48.523 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 systemd[1]: Starting Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:48.785 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 podman[49009]: 2024-05-25 06:07:48.425052922 +0000 UTC m=+0.016603825 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-25T06:07:48.785 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 podman[49009]: 2024-05-25 06:07:48.579451664 +0000 UTC m=+0.171002597 container create 3142b4d09fa063ee011d014aa7826ac67184d79bce2dfc61df3f3f205ad5ddd5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:07:49.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:07:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:07:48.741Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002899199s 2024-05-25T06:07:49.286 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 podman[49009]: 2024-05-25 06:07:48.789825188 +0000 UTC m=+0.381376086 container init 3142b4d09fa063ee011d014aa7826ac67184d79bce2dfc61df3f3f205ad5ddd5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:07:49.286 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 podman[49009]: 2024-05-25 06:07:48.792637053 +0000 UTC m=+0.384187947 container start 3142b4d09fa063ee011d014aa7826ac67184d79bce2dfc61df3f3f205ad5ddd5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:07:49.286 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.795Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-25T06:07:49.287 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.795Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-25T06:07:49.287 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-25T06:07:49.287 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-25T06:07:49.287 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-25T06:07:49.287 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:115 level=info collector=arp 2024-05-25T06:07:49.287 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-25T06:07:49.288 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-25T06:07:49.288 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-25T06:07:49.288 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-25T06:07:49.288 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-25T06:07:49.288 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.796Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-25T06:07:49.288 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-25T06:07:49.289 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-25T06:07:49.289 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=edac 2024-05-25T06:07:49.289 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-25T06:07:49.289 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-25T06:07:49.289 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-25T06:07:49.289 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-25T06:07:49.289 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-25T06:07:49.290 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-25T06:07:49.290 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-25T06:07:49.290 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-25T06:07:49.290 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-25T06:07:49.290 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-25T06:07:49.290 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-25T06:07:49.291 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-25T06:07:49.291 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-25T06:07:49.291 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-25T06:07:49.291 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-25T06:07:49.291 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-25T06:07:49.291 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=os 2024-05-25T06:07:49.292 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-25T06:07:49.292 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-25T06:07:49.292 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-25T06:07:49.292 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-25T06:07:49.292 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-25T06:07:49.292 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-25T06:07:49.293 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=stat 2024-05-25T06:07:49.293 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-25T06:07:49.293 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-25T06:07:49.293 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-25T06:07:49.293 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=time 2024-05-25T06:07:49.293 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-25T06:07:49.293 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=uname 2024-05-25T06:07:49.294 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-25T06:07:49.294 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-25T06:07:49.294 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-25T06:07:49.294 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-25T06:07:49.294 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[49024]: ts=2024-05-25T06:07:48.797Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-25T06:07:49.294 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 bash[49009]: 3142b4d09fa063ee011d014aa7826ac67184d79bce2dfc61df3f3f205ad5ddd5 2024-05-25T06:07:49.294 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:07:48 smithi179 systemd[1]: Started Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: Reconfiguring grafana.a (dependencies changed)... 2024-05-25T06:07:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-25T06:07:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T06:07:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T06:07:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:49 smithi179 ceph-mon[25623]: Reconfiguring daemon grafana.a on smithi179 2024-05-25T06:07:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: Reconfiguring grafana.a (dependencies changed)... 2024-05-25T06:07:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-25T06:07:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[24776]: Reconfiguring daemon grafana.a on smithi179 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: Reconfiguring grafana.a (dependencies changed)... 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:50.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T06:07:50.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-25T06:07:50.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:49 smithi175 ceph-mon[29436]: Reconfiguring daemon grafana.a on smithi179 2024-05-25T06:07:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:50 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:51.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:50 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:51.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:50 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:07:52.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:51 smithi179 ceph-mon[25623]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:07:52.286 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:51 smithi179 systemd[1]: Stopping Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:52.286 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:51 smithi179 bash[49194]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana.a" found: no such container 2024-05-25T06:07:52.286 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[41686]: t=2024-05-25T06:07:52+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-25T06:07:52.286 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 podman[49205]: 2024-05-25 06:07:52.042701447 +0000 UTC m=+0.136274931 container died c3a0d6b80f616bf3e31340c50a7da211a20a69d60bf98f60d73112ca930be443 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, architecture=x86_64, io.openshift.tags=base rhel8, name=ubi8, io.openshift.expose-services=, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762) 2024-05-25T06:07:52.313 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:51 smithi175 ceph-mon[24776]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:07:52.313 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:51 smithi175 ceph-mon[29436]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:07:52.691 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 podman[49205]: 2024-05-25 06:07:52.398723706 +0000 UTC m=+0.492297207 container cleanup c3a0d6b80f616bf3e31340c50a7da211a20a69d60bf98f60d73112ca930be443 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.buildah.version=1.24.2, maintainer=Paul Cuzner , vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, architecture=x86_64, name=ubi8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, build-date=2022-03-28T10:36:18.413762, com.redhat.component=ubi8-container) 2024-05-25T06:07:52.691 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 bash[49205]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a 2024-05-25T06:07:52.691 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 bash[49232]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana.a" found: no such container 2024-05-25T06:07:53.026 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 podman[49217]: 2024-05-25 06:07:52.691650369 +0000 UTC m=+0.645357431 container remove c3a0d6b80f616bf3e31340c50a7da211a20a69d60bf98f60d73112ca930be443 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, architecture=x86_64, distribution-scope=public, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.openshift.tags=base rhel8, release=236.1648460182) 2024-05-25T06:07:53.027 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-25T06:07:53.027 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 systemd[1]: Stopped Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:53.027 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@grafana.a.service: Consumed 3.265s CPU time. 2024-05-25T06:07:53.027 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 systemd[1]: Starting Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:07:53.027 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 podman[49294]: 2024-05-25 06:07:52.81930191 +0000 UTC m=+0.016713912 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-05-25T06:07:53.027 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:52 smithi179 podman[49294]: 2024-05-25 06:07:52.920335235 +0000 UTC m=+0.117747278 container create 30f86eeb19f8ff1417d05c56fa0b95eee0123d589b58e949d44292ccb9578336 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, version=8.5, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, architecture=x86_64) 2024-05-25T06:07:53.278 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 podman[49294]: 2024-05-25 06:07:53.187025622 +0000 UTC m=+0.384437609 container init 30f86eeb19f8ff1417d05c56fa0b95eee0123d589b58e949d44292ccb9578336 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, architecture=x86_64, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, build-date=2022-03-28T10:36:18.413762, name=ubi8, vcs-type=git, vendor=Red Hat, Inc.) 2024-05-25T06:07:53.278 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 podman[49294]: 2024-05-25 06:07:53.189825436 +0000 UTC m=+0.387237428 container start 30f86eeb19f8ff1417d05c56fa0b95eee0123d589b58e949d44292ccb9578336 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, io.buildah.version=1.24.2, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, release=236.1648460182, name=ubi8, vcs-type=git, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-05-25T06:07:53.278 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-05-25T06:07:53.278 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-05-25T06:07:53.278 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-05-25T06:07:53.278 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-05-25T06:07:53.279 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="App mode production" logger=settings 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=592.521µs 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-25T06:07:53.280 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 bash[49294]: 30f86eeb19f8ff1417d05c56fa0b95eee0123d589b58e949d44292ccb9578336 2024-05-25T06:07:53.281 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 systemd[1]: Started Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:07:53.281 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-25T06:07:53.281 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-25T06:07:53.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:07:53.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STOPPING 2024-05-25T06:07:53.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:07:53.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STOPPED 2024-05-25T06:07:53.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STARTING 2024-05-25T06:07:53.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Serving on http://:::9283 2024-05-25T06:07:53.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STARTED 2024-05-25T06:07:53.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STOPPING 2024-05-25T06:07:53.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:07:53.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STOPPED 2024-05-25T06:07:53.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STARTING 2024-05-25T06:07:53.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:07:53] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-05-25T06:07:53.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-25T06:07:53.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-05-25T06:07:53.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-25T06:07:53.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-05-25T06:07:53.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-25T06:07:53.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-25T06:07:53.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:07:53+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-25T06:07:54.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Serving on http://:::9283 2024-05-25T06:07:54.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STARTED 2024-05-25T06:07:54.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STOPPING 2024-05-25T06:07:54.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:07:54.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STOPPED 2024-05-25T06:07:54.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STARTING 2024-05-25T06:07:54.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Serving on http://:::9283 2024-05-25T06:07:54.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:07:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:07:53] ENGINE Bus STARTED 2024-05-25T06:07:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:07:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:07:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:07:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi175.front.sepia.ceph.com:9093"}]: dispatch 2024-05-25T06:07:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi175.front.sepia.ceph.com:9093"}]: dispatch 2024-05-25T06:07:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:07:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:07:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-05-25T06:07:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:07:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/816880151' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:07:54.573 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/110643839' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3840021495"}]: dispatch 2024-05-25T06:07:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:07:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi175.front.sepia.ceph.com:9093"}]: dispatch 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi175.front.sepia.ceph.com:9093"}]: dispatch 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-05-25T06:07:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/816880151' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:07:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:54 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/110643839' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3840021495"}]: dispatch 2024-05-25T06:07:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:07:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:07:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:07:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi175.front.sepia.ceph.com:9093"}]: dispatch 2024-05-25T06:07:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi175.front.sepia.ceph.com:9093"}]: dispatch 2024-05-25T06:07:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:07:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:07:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-05-25T06:07:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-05-25T06:07:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:07:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:07:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-05-25T06:07:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-05-25T06:07:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:07:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/816880151' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:07:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:54 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/110643839' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3840021495"}]: dispatch 2024-05-25T06:07:55.723 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:55 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/110643839' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3840021495"}]': finished 2024-05-25T06:07:55.723 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:55 smithi175 ceph-mon[24776]: osdmap e98: 8 total, 8 up, 8 in 2024-05-25T06:07:55.723 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:55 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1588062467' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4130354415"}]: dispatch 2024-05-25T06:07:55.723 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:55 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/110643839' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3840021495"}]': finished 2024-05-25T06:07:55.723 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:55 smithi175 ceph-mon[29436]: osdmap e98: 8 total, 8 up, 8 in 2024-05-25T06:07:55.723 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:55 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1588062467' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4130354415"}]: dispatch 2024-05-25T06:07:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:55 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/110643839' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3840021495"}]': finished 2024-05-25T06:07:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:55 smithi179 ceph-mon[25623]: osdmap e98: 8 total, 8 up, 8 in 2024-05-25T06:07:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:55 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1588062467' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4130354415"}]: dispatch 2024-05-25T06:07:56.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[29436]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1588062467' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4130354415"}]': finished 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[29436]: osdmap e99: 8 total, 8 up, 8 in 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/533241164' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]: dispatch 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]: dispatch 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[24776]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1588062467' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4130354415"}]': finished 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[24776]: osdmap e99: 8 total, 8 up, 8 in 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/533241164' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]: dispatch 2024-05-25T06:07:56.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:56 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]: dispatch 2024-05-25T06:07:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:56 smithi179 ceph-mon[25623]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:56 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1588062467' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4130354415"}]': finished 2024-05-25T06:07:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:56 smithi179 ceph-mon[25623]: osdmap e99: 8 total, 8 up, 8 in 2024-05-25T06:07:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:56 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/533241164' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]: dispatch 2024-05-25T06:07:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:56 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]: dispatch 2024-05-25T06:07:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:57 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]': finished 2024-05-25T06:07:57.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:57 smithi179 ceph-mon[25623]: osdmap e100: 8 total, 8 up, 8 in 2024-05-25T06:07:57.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:57 smithi179 ceph-mon[25623]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:57.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:57 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:57.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:57 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:57.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:57 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3288847694' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]: dispatch 2024-05-25T06:07:57.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:57 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]: dispatch 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]': finished 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[29436]: osdmap e100: 8 total, 8 up, 8 in 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[29436]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3288847694' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]: dispatch 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]: dispatch 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/994931434"}]': finished 2024-05-25T06:07:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[24776]: osdmap e100: 8 total, 8 up, 8 in 2024-05-25T06:07:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[24776]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:07:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3288847694' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]: dispatch 2024-05-25T06:07:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:57 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]: dispatch 2024-05-25T06:07:58.956 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]': finished 2024-05-25T06:07:58.956 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: osdmap e101: 8 total, 8 up, 8 in 2024-05-25T06:07:58.956 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:58.956 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:58.957 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:58.957 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:07:58.957 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:58.957 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:07:58.957 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:58.957 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:58.958 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:58.958 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:07:58.958 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/4158913760' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]: dispatch 2024-05-25T06:07:58.958 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]: dispatch 2024-05-25T06:07:58.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]': finished 2024-05-25T06:07:58.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: osdmap e101: 8 total, 8 up, 8 in 2024-05-25T06:07:58.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:58.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:58.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:58.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:07:58.960 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:58.960 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:07:58.960 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:58.960 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:58.960 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:58.960 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:07:58.961 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/4158913760' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]: dispatch 2024-05-25T06:07:58.961 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:58 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]: dispatch 2024-05-25T06:07:59.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:07:58 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:07:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:07:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3065084381"}]': finished 2024-05-25T06:07:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: osdmap e101: 8 total, 8 up, 8 in 2024-05-25T06:07:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/4158913760' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]: dispatch 2024-05-25T06:07:59.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:58 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]: dispatch 2024-05-25T06:08:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:59 smithi179 ceph-mon[25623]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 1 op/s 2024-05-25T06:08:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:59 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]': finished 2024-05-25T06:08:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:59 smithi179 ceph-mon[25623]: osdmap e102: 8 total, 8 up, 8 in 2024-05-25T06:08:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:59 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/885913901' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]: dispatch 2024-05-25T06:08:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:07:59 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]: dispatch 2024-05-25T06:08:00.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[24776]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 1 op/s 2024-05-25T06:08:00.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]': finished 2024-05-25T06:08:00.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[24776]: osdmap e102: 8 total, 8 up, 8 in 2024-05-25T06:08:00.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/885913901' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]: dispatch 2024-05-25T06:08:00.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]: dispatch 2024-05-25T06:08:00.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[29436]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 1 op/s 2024-05-25T06:08:00.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/1129599620"}]': finished 2024-05-25T06:08:00.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[29436]: osdmap e102: 8 total, 8 up, 8 in 2024-05-25T06:08:00.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/885913901' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]: dispatch 2024-05-25T06:08:00.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:07:59 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]: dispatch 2024-05-25T06:08:01.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:00 smithi175 ceph-mon[24776]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]': finished 2024-05-25T06:08:01.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:00 smithi175 ceph-mon[24776]: osdmap e103: 8 total, 8 up, 8 in 2024-05-25T06:08:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:00 smithi175 ceph-mon[29436]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]': finished 2024-05-25T06:08:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:00 smithi175 ceph-mon[29436]: osdmap e103: 8 total, 8 up, 8 in 2024-05-25T06:08:01.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:00 smithi179 ceph-mon[25623]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/1129599620"}]': finished 2024-05-25T06:08:01.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:00 smithi179 ceph-mon[25623]: osdmap e103: 8 total, 8 up, 8 in 2024-05-25T06:08:01.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:08:01.163+0000 7fa9cb44e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:08:01.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:01.287 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return future.result(timeout) 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.__get_result() 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: raise self._exception 2024-05-25T06:08:01.288 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: use_current_daemon_image=reconfig, 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:08:01.289 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:01.290 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:01 smithi179 ceph-mon[25623]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 1 op/s 2024-05-25T06:08:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:01 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:08:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:08:02.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:02.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:02.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:02.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:02.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:02.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:01 smithi175 ceph-mon[24776]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 1 op/s 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:01 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:02.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:08:02.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:02.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:01 smithi175 ceph-mon[29436]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 1 op/s 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:01 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:08:02.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:08:02.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:08:02.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:08:02.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:08:02.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:08:03.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:08:03] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-05-25T06:08:03.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:04.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:03 smithi179 ceph-mon[25623]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:08:04.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:03 smithi175 ceph-mon[24776]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:08:04.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:03 smithi175 ceph-mon[29436]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:08:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:06 smithi179 ceph-mon[25623]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:08:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:06 smithi175 ceph-mon[24776]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:08:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:06 smithi175 ceph-mon[29436]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:08:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:08 smithi179 ceph-mon[25623]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-25T06:08:08.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:08.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:08 smithi175 ceph-mon[24776]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-25T06:08:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:08 smithi175 ceph-mon[29436]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-25T06:08:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:10 smithi179 ceph-mon[25623]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 427 B/s rd, 0 op/s 2024-05-25T06:08:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:10 smithi175 ceph-mon[24776]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 427 B/s rd, 0 op/s 2024-05-25T06:08:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:10 smithi175 ceph-mon[29436]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 427 B/s rd, 0 op/s 2024-05-25T06:08:11.576 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:11 smithi175 ceph-mon[24776]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 387 B/s rd, 0 op/s 2024-05-25T06:08:11.577 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:11 smithi175 ceph-mon[29436]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 387 B/s rd, 0 op/s 2024-05-25T06:08:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:11 smithi179 ceph-mon[25623]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 387 B/s rd, 0 op/s 2024-05-25T06:08:13.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:08:13] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-25T06:08:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:13 smithi179 ceph-mon[25623]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-25T06:08:13.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:13 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2184093667' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:08:13.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:13 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3404606334' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4138469446"}]: dispatch 2024-05-25T06:08:13.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:13 smithi175 ceph-mon[24776]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-25T06:08:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:13 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2184093667' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:08:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:13 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3404606334' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4138469446"}]: dispatch 2024-05-25T06:08:14.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:13 smithi175 ceph-mon[29436]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-25T06:08:14.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:13 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2184093667' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:08:14.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:13 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3404606334' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4138469446"}]: dispatch 2024-05-25T06:08:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:14 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3404606334' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4138469446"}]': finished 2024-05-25T06:08:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:14 smithi179 ceph-mon[25623]: osdmap e104: 8 total, 8 up, 8 in 2024-05-25T06:08:15.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:14 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3404606334' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4138469446"}]': finished 2024-05-25T06:08:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:14 smithi175 ceph-mon[24776]: osdmap e104: 8 total, 8 up, 8 in 2024-05-25T06:08:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:14 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3404606334' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4138469446"}]': finished 2024-05-25T06:08:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:14 smithi175 ceph-mon[29436]: osdmap e104: 8 total, 8 up, 8 in 2024-05-25T06:08:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:16 smithi179 ceph-mon[25623]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:08:16.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:16 smithi175 ceph-mon[24776]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:08:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:16 smithi175 ceph-mon[29436]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:08:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:18 smithi179 ceph-mon[25623]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:08:18.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:18.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:18 smithi175 ceph-mon[24776]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:08:18.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:18 smithi175 ceph-mon[29436]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:08:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:20 smithi179 ceph-mon[25623]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-25T06:08:20.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:20 smithi175 ceph-mon[24776]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-25T06:08:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:20 smithi175 ceph-mon[29436]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-25T06:08:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:21 smithi179 ceph-mon[25623]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-25T06:08:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:21.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:21 smithi175 ceph-mon[24776]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-25T06:08:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:21 smithi175 ceph-mon[29436]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-25T06:08:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:23 smithi179 ceph-mon[25623]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-25T06:08:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:23 smithi179 ceph-mon[25623]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:08:23] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-25T06:08:23.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:23 smithi175 ceph-mon[24776]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-25T06:08:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:23 smithi175 ceph-mon[24776]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:23 smithi175 ceph-mon[29436]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-25T06:08:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:23 smithi175 ceph-mon[29436]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:26 smithi179 ceph-mon[25623]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-05-25T06:08:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:26 smithi175 ceph-mon[24776]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-05-25T06:08:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:26 smithi175 ceph-mon[29436]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-05-25T06:08:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:28 smithi179 ceph-mon[25623]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:08:28.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:28.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:28 smithi175 ceph-mon[24776]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:08:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:28 smithi175 ceph-mon[29436]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:08:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:30 smithi179 ceph-mon[25623]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:30.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:30 smithi175 ceph-mon[24776]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:30 smithi175 ceph-mon[29436]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:32 smithi179 ceph-mon[25623]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:32.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:32 smithi175 ceph-mon[24776]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:32.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:32 smithi175 ceph-mon[29436]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:33 smithi179 ceph-mon[25623]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:33 smithi179 ceph-mon[25623]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:08:33] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-25T06:08:33.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:33.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:33 smithi175 ceph-mon[29436]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:33.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:33 smithi175 ceph-mon[29436]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:33 smithi175 ceph-mon[24776]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:33 smithi175 ceph-mon[24776]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:36 smithi179 ceph-mon[25623]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:36 smithi175 ceph-mon[29436]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:36 smithi175 ceph-mon[24776]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:38 smithi179 ceph-mon[25623]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:38.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:38 smithi175 ceph-mon[24776]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:38 smithi175 ceph-mon[29436]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:40 smithi179 ceph-mon[25623]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:40 smithi175 ceph-mon[29436]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:40 smithi175 ceph-mon[24776]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:42 smithi179 ceph-mon[25623]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:42.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:42 smithi175 ceph-mon[29436]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:42 smithi175 ceph-mon[24776]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:43 smithi179 ceph-mon[25623]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:43 smithi179 ceph-mon[25623]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:43.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:08:43] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-25T06:08:43.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:43.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:43 smithi175 ceph-mon[29436]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:43 smithi175 ceph-mon[29436]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:43 smithi175 ceph-mon[24776]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:43 smithi175 ceph-mon[24776]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:46 smithi179 ceph-mon[25623]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:46.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:46 smithi175 ceph-mon[29436]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:46 smithi175 ceph-mon[24776]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:47 smithi175 ceph-mon[29436]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:47 smithi175 ceph-mon[24776]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:47 smithi179 ceph-mon[25623]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:49.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:50 smithi179 ceph-mon[25623]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:50.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:50 smithi175 ceph-mon[29436]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:50 smithi175 ceph-mon[24776]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:51 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:51.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:51 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:51 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:08:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:52 smithi179 ceph-mon[25623]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:52 smithi175 ceph-mon[29436]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:52 smithi175 ceph-mon[24776]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:53 smithi179 ceph-mon[25623]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:53 smithi179 ceph-mon[25623]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:53.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:08:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:08:53] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-25T06:08:53.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:08:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:53 smithi175 ceph-mon[29436]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:53 smithi175 ceph-mon[29436]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:53 smithi175 ceph-mon[24776]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:53 smithi175 ceph-mon[24776]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:08:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:56 smithi179 ceph-mon[25623]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:56 smithi175 ceph-mon[29436]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:56 smithi175 ceph-mon[24776]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:08:57.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:08:57 smithi175 ceph-mon[24776]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:08:57 smithi175 ceph-mon[29436]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:08:57 smithi179 ceph-mon[25623]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:08:59.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:08:58 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:08:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:00 smithi179 ceph-mon[25623]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:00 smithi175 ceph-mon[24776]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:00 smithi175 ceph-mon[29436]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:01 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:09:01.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:01 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:09:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:01 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:09:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:02 smithi179 ceph-mon[25623]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:02.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:02 smithi175 ceph-mon[24776]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:02 smithi175 ceph-mon[29436]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:03.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:09:03] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-25T06:09:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:03 smithi179 ceph-mon[25623]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:03 smithi179 ceph-mon[25623]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:03.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:03.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:03 smithi175 ceph-mon[29436]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:03.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:03 smithi175 ceph-mon[29436]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:03.812 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:03 smithi175 ceph-mon[24776]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:03.812 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:03 smithi175 ceph-mon[24776]: from='client.15144 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:05 smithi179 ceph-mon[25623]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:05 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:05 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:05 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:05 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:05 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[29436]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[24776]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:06.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:05 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:09:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:09:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:09:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:09:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:09:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:09:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:06 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:09:06.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:09:06.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:09:06.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:09:06.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:09:06.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:09:06.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:09:06.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:09:06.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:06 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:09:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:08 smithi179 ceph-mon[25623]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:08.551 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:08 smithi175 ceph-mon[24776]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:08.552 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:08 smithi175 ceph-mon[29436]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:09.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:09.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:09:09.238+0000 7fa9cb44e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:09.536 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:09.537 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return future.result(timeout) 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:09:09.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.__get_result() 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: raise self._exception 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: use_current_daemon_image=reconfig, 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:09.539 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:09.540 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:10 smithi179 ceph-mon[25623]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:10 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:09:10.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:10 smithi175 ceph-mon[29436]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:10.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:10 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:10.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:09:10.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:10.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:10.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:10.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:10.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:09:10.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:09:10.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.826 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:10 smithi175 ceph-mon[24776]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:10.827 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:10 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:10.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:09:10.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:09:10.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:09:10.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:09:10.830 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:09:10.831 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:09:10.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:09:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:11 smithi179 ceph-mon[25623]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:11.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:11 smithi175 ceph-mon[24776]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:11.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:11 smithi175 ceph-mon[29436]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:13 smithi179 ceph-mon[25623]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:09:13.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:09:13] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-25T06:09:13.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:13 smithi175 ceph-mon[24776]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:09:14.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:13 smithi175 ceph-mon[29436]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:09:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:16 smithi179 ceph-mon[25623]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-25T06:09:16.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:16 smithi175 ceph-mon[24776]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-25T06:09:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:16 smithi175 ceph-mon[29436]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-25T06:09:17.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:17 smithi179 ceph-mon[25623]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-25T06:09:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:17 smithi175 ceph-mon[24776]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-25T06:09:17.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:17 smithi175 ceph-mon[29436]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-25T06:09:19.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:20.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:20 smithi175 ceph-mon[24776]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-25T06:09:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:20 smithi175 ceph-mon[29436]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-25T06:09:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:20 smithi179 ceph-mon[25623]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-25T06:09:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:21 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3591670349' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:09:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:21 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3591670349' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:09:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:21 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3591670349' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:09:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:22 smithi179 ceph-mon[25623]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-25T06:09:22.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:22 smithi175 ceph-mon[24776]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-25T06:09:22.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:22 smithi175 ceph-mon[29436]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-25T06:09:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:23 smithi179 ceph-mon[25623]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:09:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:09:23] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-25T06:09:23.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:23.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:23 smithi175 ceph-mon[24776]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:09:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:23 smithi175 ceph-mon[29436]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:09:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:26 smithi179 ceph-mon[25623]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:09:26.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:26 smithi175 ceph-mon[24776]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:09:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:26 smithi175 ceph-mon[29436]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:09:27.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:27 smithi179 ceph-mon[25623]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:27.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:27 smithi175 ceph-mon[29436]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:27 smithi175 ceph-mon[24776]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:29.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:30 smithi179 ceph-mon[25623]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:30.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:30 smithi175 ceph-mon[24776]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:30 smithi175 ceph-mon[29436]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:31.502 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:31 smithi175 ceph-mon[24776]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:31.502 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:31 smithi175 ceph-mon[24776]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:31.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:31 smithi179 ceph-mon[25623]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:31.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:31 smithi179 ceph-mon[25623]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:31 smithi175 ceph-mon[29436]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:31 smithi175 ceph-mon[29436]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:09:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:33 smithi179 ceph-mon[25623]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-25T06:09:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:09:33] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-25T06:09:33.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:33 smithi175 ceph-mon[24776]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-25T06:09:34.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:33 smithi175 ceph-mon[29436]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-25T06:09:35.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:35 smithi175 ceph-mon[24776]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:35 smithi175 ceph-mon[29436]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:35 smithi179 ceph-mon[25623]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:36.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:37.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:37 smithi175 ceph-mon[24776]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:37 smithi175 ceph-mon[29436]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:37 smithi179 ceph-mon[25623]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:39.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:40 smithi179 ceph-mon[25623]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:40.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:40 smithi175 ceph-mon[29436]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:40 smithi175 ceph-mon[24776]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:41 smithi179 ceph-mon[25623]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:41 smithi179 ceph-mon[25623]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:41.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:41 smithi175 ceph-mon[24776]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:41.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:41 smithi175 ceph-mon[24776]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:41.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:41 smithi175 ceph-mon[29436]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:41.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:41 smithi175 ceph-mon[29436]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:43 smithi179 ceph-mon[25623]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:43.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:09:43] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-25T06:09:43.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:43 smithi175 ceph-mon[29436]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:43 smithi175 ceph-mon[24776]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:45.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:45 smithi175 ceph-mon[24776]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:45 smithi175 ceph-mon[29436]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:45 smithi179 ceph-mon[25623]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:47 smithi179 ceph-mon[25623]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:48.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:47 smithi175 ceph-mon[24776]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:47 smithi175 ceph-mon[29436]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:49.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:50.773 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:50 smithi175 ceph-mon[24776]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:50.773 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:50 smithi175 ceph-mon[29436]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:50 smithi179 ceph-mon[25623]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:51 smithi179 ceph-mon[25623]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:51 smithi179 ceph-mon[25623]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:51 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:51.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:51 smithi175 ceph-mon[24776]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:51.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:51 smithi175 ceph-mon[24776]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:51.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:51 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:51 smithi175 ceph-mon[29436]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:09:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:51 smithi175 ceph-mon[29436]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:51 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:09:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:53 smithi179 ceph-mon[25623]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:53.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:09:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:09:53] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-05-25T06:09:53.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:09:54.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:53 smithi175 ceph-mon[29436]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:53 smithi175 ceph-mon[24776]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:55.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:55 smithi175 ceph-mon[29436]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:55 smithi175 ceph-mon[24776]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:55 smithi179 ceph-mon[25623]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:09:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:09:57 smithi179 ceph-mon[25623]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:58.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:09:57 smithi175 ceph-mon[24776]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:09:57 smithi175 ceph-mon[29436]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:09:59.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:09:58 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:09:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:00 smithi179 ceph-mon[25623]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:00 smithi179 ceph-mon[25623]: overall HEALTH_OK 2024-05-25T06:10:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:00 smithi175 ceph-mon[24776]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:00 smithi175 ceph-mon[24776]: overall HEALTH_OK 2024-05-25T06:10:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:00 smithi175 ceph-mon[29436]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:00 smithi175 ceph-mon[29436]: overall HEALTH_OK 2024-05-25T06:10:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:01 smithi179 ceph-mon[25623]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:01 smithi179 ceph-mon[25623]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:01 smithi175 ceph-mon[24776]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:01 smithi175 ceph-mon[24776]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:01 smithi175 ceph-mon[29436]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:01 smithi175 ceph-mon[29436]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:03 smithi179 ceph-mon[25623]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:03.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:10:03] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-05-25T06:10:03.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:03 smithi175 ceph-mon[29436]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:04.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:03 smithi175 ceph-mon[24776]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:05.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:05 smithi175 ceph-mon[24776]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:05 smithi175 ceph-mon[29436]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:05 smithi179 ceph-mon[25623]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:06.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:07.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:07 smithi175 ceph-mon[29436]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:07 smithi175 ceph-mon[24776]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:07 smithi179 ceph-mon[25623]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:09.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:09 smithi179 ceph-mon[25623]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:09 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:10.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:09 smithi175 ceph-mon[29436]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:09 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:09 smithi175 ceph-mon[24776]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:09 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:10 smithi175 ceph-mon[24776]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:10 smithi175 ceph-mon[29436]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:10.905 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:10 smithi179 ceph-mon[25623]: from='client.25060 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:11 smithi179 ceph-mon[25623]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:11 smithi175 ceph-mon[24776]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:11 smithi175 ceph-mon[29436]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:13 smithi179 ceph-mon[25623]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:13 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:13 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:13.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:10:13] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-05-25T06:10:13.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:13 smithi175 ceph-mon[24776]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:13 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:13 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:14.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:13 smithi175 ceph-mon[29436]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:14.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:13 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:14.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:13 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:14.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:14.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:14.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:14.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:10:14.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:14.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:14.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:14.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:14.943 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:14 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:14 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:10:15.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:15 smithi175 ceph-mon[24776]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:15.959 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:15 smithi175 ceph-mon[29436]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:15 smithi179 ceph-mon[25623]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:17.649 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:10:17.367+0000 7fa9cb44e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:17.649 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:10:17.649 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:17.649 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:17.649 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:17.650 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:17.651 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:17.651 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:17.651 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:17.651 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:10:17.651 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:10:17.651 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:10:17.651 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:10:17.652 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:10:17.652 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:10:17.652 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:10:17.652 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:10:17.652 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return future.result(timeout) 2024-05-25T06:10:17.652 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:10:17.652 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return self.__get_result() 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: raise self._exception 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: use_current_daemon_image=reconfig, 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Creating ceph-iscsi config... 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:17.653 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: Traceback (most recent call last): 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: return _run_code(code, main_globals, None, 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: exec(code, run_globals) 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:17.654 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:17.655 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:17 smithi179 ceph-mon[25623]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:17 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:10:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:10:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:18.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:18.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:18.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:18.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:18.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:17 smithi175 ceph-mon[24776]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:17 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:10:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:10:18.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:10:18.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:10:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:10:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:10:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:17 smithi175 ceph-mon[29436]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:18.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:17 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:18.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:10:18.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:18.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:18.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:18.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:18.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:10:18.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:18.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:10:18.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:18.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:18.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:18.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:18.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:18.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:18.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:18.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:10:18.083 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:10:18.083 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:10:18.083 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:10:18.083 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:10:18.083 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:10:18.083 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:10:18.084 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:10:18.084 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:10:18.084 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:10:18.084 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:10:18.084 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:10:18.085 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:10:18.085 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:10:18.085 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:10:18.085 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:10:18.085 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:10:18.085 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:10:18.085 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:10:18.086 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:10:18.086 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:10:18.086 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.086 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:18.087 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:10:18.087 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:10:18.087 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:10:18.087 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:10:18.087 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:10:18.088 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:10:18.088 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:10:18.088 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:10:18.088 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:10:18.088 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:10:18.088 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:10:18.088 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:10:18.089 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:10:18.089 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:10:19.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:20.773 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:20 smithi175 ceph-mon[24776]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:20.773 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:20 smithi175 ceph-mon[29436]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:20 smithi179 ceph-mon[25623]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:21 smithi179 ceph-mon[25623]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:21 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:21.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:21 smithi175 ceph-mon[24776]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:21 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:21 smithi175 ceph-mon[29436]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:21 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:23.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-25T06:10:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:23 smithi179 ceph-mon[25623]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:23.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:23.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:10:23] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-05-25T06:10:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:23 smithi175 ceph-mon[24776]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:23 smithi175 ceph-mon[29436]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:10:25.599 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled to redeploy mgr.y on host 'smithi175' 2024-05-25T06:10:25.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:25 smithi179 ceph-mon[25623]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:10:25.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:25 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3640060856' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T06:10:25.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:25 smithi175 ceph-mon[24776]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:10:25.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:25 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3640060856' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T06:10:25.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:25 smithi175 ceph-mon[29436]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:10:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:25 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3640060856' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-25T06:10:26.245 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps --refresh' 2024-05-25T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[29436]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[29436]: Schedule redeploy daemon mgr.y 2024-05-25T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:26.912 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:26.912 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:26.912 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[24776]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:10:26.912 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:26.912 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[24776]: Schedule redeploy daemon mgr.y 2024-05-25T06:10:26.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:26.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:26.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:26 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:26 smithi179 ceph-mon[25623]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:10:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:26 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:26 smithi179 ceph-mon[25623]: Schedule redeploy daemon mgr.y 2024-05-25T06:10:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:26 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:26 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:26 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:27.890 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:27 smithi175 ceph-mon[29436]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:10:27.891 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:27 smithi175 ceph-mon[24776]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:10:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:27 smithi179 ceph-mon[25623]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:10:28.402 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:10:28.402 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (2m) 83s ago 10m 13.8M - 0.23.0 ba2b418f427c 1893f9da103a 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (2m) 14s ago 10m 46.2M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (2m) 83s ago 9m 48.2M - 3.5 e1d6a67b021e 4ae398f3b6ad 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283 running (5m) 14s ago 15m 509M - 19.0.0-2843-gea0684c0 f9a9809cafd6 2c455cbdf457 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:9283 running (16m) 83s ago 16m 409M - 17.2.0 e1d6a67b021e 8822d7e4e7de 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (16m) 83s ago 16m 55.8M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (15m) 14s ago 15m 46.6M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (15m) 83s ago 15m 45.5M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (2m) 83s ago 11m 18.5M - 1.3.1 1dbe0e931976 3fd7592e3fcc 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (2m) 14s ago 11m 8908k - 1.3.1 1dbe0e931976 3142b4d09fa0 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (14m) 83s ago 14m 54.4M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (14m) 83s ago 14m 55.7M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:10:28.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (14m) 83s ago 14m 50.8M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (13m) 83s ago 13m 51.2M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (13m) 14s ago 13m 51.7M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (12m) 14s ago 12m 50.3M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (12m) 14s ago 12m 51.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (12m) 14s ago 12m 53.4M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (2m) 14s ago 11m 72.1M - 2.33.4 514e6a882f6e b84f89714aa4 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (9m) 83s ago 9m 72.2M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:10:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (9m) 14s ago 9m 75.7M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:10:28.718 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:28 smithi175 ceph-mon[24776]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:10:28.718 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:28 smithi175 ceph-mon[29436]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:10:28.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'sleep 180' 2024-05-25T06:10:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:28 smithi179 ceph-mon[25623]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:10:29.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:29.933 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:29 smithi175 ceph-mon[29436]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:29.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:29 smithi175 ceph-mon[24776]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:30.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:29 smithi179 ceph-mon[25623]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:30 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:31.250 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:30 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:31.250 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:30 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:32.124 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:31 smithi175 ceph-mon[29436]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:32.124 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:31 smithi175 ceph-mon[24776]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:32.259 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:31 smithi179 ceph-mon[25623]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:33.205 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:32 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2642144357' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:10:33.206 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:32 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2642144357' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:10:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:32 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2642144357' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:10:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:10:33] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-05-25T06:10:33.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:33.503Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:34.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:33 smithi175 ceph-mon[24776]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:34.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:33 smithi175 ceph-mon[29436]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:33 smithi179 ceph-mon[25623]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:35 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:35 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:35 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:35 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:35 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:35 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:35 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:35 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:36.171 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:36 smithi179 ceph-mon[25623]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:36.171 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:10:36.171 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:36 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:10:36.171 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:10:36.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[29436]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:36.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:10:36.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:10:36.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:10:36.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[29436]: Deploying daemon mgr.y on smithi175 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[24776]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[24776]: Deploying daemon mgr.y on smithi175 2024-05-25T06:10:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:36 smithi179 ceph-mon[25623]: Deploying daemon mgr.y on smithi175 2024-05-25T06:10:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:37.164 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:36 smithi175 systemd[1]: Stopping Ceph mgr.y for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:10:37.425 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 podman[63130]: 2024-05-25 06:10:37.164457283 +0000 UTC m=+0.151965844 container died 8822d7e4e7de28de687190863ea9249b20fff866cda3057f4daf4066d96a331f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., version=8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, architecture=x86_64, ceph=True, GIT_CLEAN=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-25T06:10:37.676 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 podman[63130]: 2024-05-25 06:10:37.482861863 +0000 UTC m=+0.470370416 container cleanup 8822d7e4e7de28de687190863ea9249b20fff866cda3057f4daf4066d96a331f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, RELEASE=HEAD, ceph=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, release=754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git) 2024-05-25T06:10:37.676 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 bash[63130]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y 2024-05-25T06:10:37.928 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 podman[63142]: 2024-05-25 06:10:37.676388165 +0000 UTC m=+0.515995293 container remove 8822d7e4e7de28de687190863ea9249b20fff866cda3057f4daf4066d96a331f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-25T06:10:37.928 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:10:37.928 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-05-25T06:10:37.928 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 systemd[1]: Stopped Ceph mgr.y for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:10:37.928 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:37 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service: Consumed 1min 1.518s CPU time. 2024-05-25T06:10:38.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:38 smithi175 ceph-mon[24776]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:38.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 systemd[1]: Starting Ceph mgr.y for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:10:38.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:38 smithi175 ceph-mon[29436]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:38 smithi179 ceph-mon[25623]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:38.613 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 podman[63269]: 2024-05-25 06:10:38.230594903 +0000 UTC m=+0.017200277 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:10:38.614 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 podman[63269]: 2024-05-25 06:10:38.386187441 +0000 UTC m=+0.172792802 container create a0734416908919eacc2c8841632da2b866067412f28dba3a6dbd76d5782193ff (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-25T06:10:38.969 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 podman[63269]: 2024-05-25 06:10:38.614029005 +0000 UTC m=+0.400634366 container init a0734416908919eacc2c8841632da2b866067412f28dba3a6dbd76d5782193ff (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-25T06:10:38.969 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 podman[63269]: 2024-05-25 06:10:38.616915643 +0000 UTC m=+0.403521005 container start a0734416908919eacc2c8841632da2b866067412f28dba3a6dbd76d5782193ff (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-25T06:10:38.970 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 ceph-mgr[63288]: -- 172.21.15.175:0/2768113571 <== mon.1 v2:172.21.15.175:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564bbef1c5a0 con 0x564bbeef1000 2024-05-25T06:10:38.970 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 bash[63269]: a0734416908919eacc2c8841632da2b866067412f28dba3a6dbd76d5782193ff 2024-05-25T06:10:38.970 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 systemd[1]: Started Ceph mgr.y for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:10:38.970 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:38.772+0000 7fe2c1d55200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:10:38.970 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:38.968+0000 7fe2c1d55200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:10:39.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:39.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:39.076+0000 7fe2c1d55200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:10:39.733 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:39.404+0000 7fe2c1d55200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:10:39.733 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:39 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:39.492+0000 7fe2c1d55200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:10:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:10:40.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:40.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:40.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:40.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:39 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:40.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:39 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:10:41.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:40.740+0000 7fe2c1d55200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:10:41.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:40.916+0000 7fe2c1d55200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:10:41.425 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:41.112+0000 7fe2c1d55200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:10:41.742 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:41.431+0000 7fe2c1d55200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:10:41.743 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:41.527+0000 7fe2c1d55200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:10:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:41 smithi179 ceph-mon[25623]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:41 smithi175 ceph-mon[29436]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:41 smithi175 ceph-mon[24776]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:42.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:41.742+0000 7fe2c1d55200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:10:42.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:41.943+0000 7fe2c1d55200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:10:42.684 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:42.611+0000 7fe2c1d55200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:10:43.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:43.064+0000 7fe2c1d55200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:10:43.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:43.172+0000 7fe2c1d55200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:10:43.687 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:43.414+0000 7fe2c1d55200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:10:43.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:43.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:10:43] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-05-25T06:10:44.255 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:43.994+0000 7fe2c1d55200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:10:44.255 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:44.087+0000 7fe2c1d55200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:10:44.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:44.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/644206147' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:10:44.256 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:10:44.257 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:10:44.257 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:44.257 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:44 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/644206147' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:10:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:10:44.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:10:44.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:10:44.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:10:44.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:10:44.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:44 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/644206147' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:10:44.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:44.356+0000 7fe2c1d55200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:10:44.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:44.495+0000 7fe2c1d55200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:10:44.854 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:44.602+0000 7fe2c1d55200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:10:44.854 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:44.723+0000 7fe2c1d55200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:10:45.120 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:45.040+0000 7fe2c1d55200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:10:45.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:45 smithi179 ceph-mon[25623]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:45.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:45 smithi175 ceph-mon[29436]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:45 smithi175 ceph-mon[24776]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:45.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:45.695+0000 7fe2c1d55200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:10:46.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:45.922+0000 7fe2c1d55200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:10:46.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:10:45] ENGINE Bus STARTING 2024-05-25T06:10:46.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CherryPy Checker: 2024-05-25T06:10:46.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: The Application mounted at '' has an empty config. 2024-05-25T06:10:46.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:10:46.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:10:46] ENGINE Serving on http://:::9283 2024-05-25T06:10:46.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:10:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:10:46] ENGINE Bus STARTED 2024-05-25T06:10:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:46 smithi179 ceph-mon[25623]: Standby manager daemon y restarted 2024-05-25T06:10:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:46 smithi179 ceph-mon[25623]: Standby manager daemon y started 2024-05-25T06:10:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:46 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:10:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:46 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:10:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:46 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:10:46.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:46 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:10:46.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[29436]: Standby manager daemon y restarted 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[29436]: Standby manager daemon y started 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[24776]: Standby manager daemon y restarted 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[24776]: Standby manager daemon y started 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:10:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:46 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:10:47.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:47 smithi179 ceph-mon[25623]: mgrmap e26: x(active, since 3m), standbys: y 2024-05-25T06:10:47.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:47 smithi179 ceph-mon[25623]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:47.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:47 smithi175 ceph-mon[29436]: mgrmap e26: x(active, since 3m), standbys: y 2024-05-25T06:10:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:47 smithi175 ceph-mon[29436]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:47 smithi175 ceph-mon[24776]: mgrmap e26: x(active, since 3m), standbys: y 2024-05-25T06:10:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:47 smithi175 ceph-mon[24776]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:10:49.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:48 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:48 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:48 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:48 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:48 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:10:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:48 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:10:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:49 smithi179 ceph-mon[25623]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:50.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:49 smithi175 ceph-mon[24776]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:50.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:49 smithi175 ceph-mon[29436]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:50.971 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:50 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:50.971 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:50 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:50 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:10:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:51 smithi179 ceph-mon[25623]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:52.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:51 smithi175 ceph-mon[24776]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:51 smithi175 ceph-mon[29436]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:53.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:10:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:10:53] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-05-25T06:10:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:53 smithi179 ceph-mon[25623]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:53 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:53.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:10:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:53 smithi175 ceph-mon[24776]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:53 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:53 smithi175 ceph-mon[29436]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:10:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:53 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:10:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:55 smithi179 ceph-mon[25623]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:10:55.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:55 smithi175 ceph-mon[29436]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:10:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:55 smithi175 ceph-mon[24776]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:10:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:57 smithi179 ceph-mon[25623]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:58.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:57 smithi175 ceph-mon[24776]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:57 smithi175 ceph-mon[29436]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:10:59.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:10:58 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:10:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:10:59 smithi179 ceph-mon[25623]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:00.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:10:59 smithi175 ceph-mon[24776]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:10:59 smithi175 ceph-mon[29436]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:01 smithi179 ceph-mon[25623]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:01 smithi175 ceph-mon[24776]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:01 smithi175 ceph-mon[29436]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:03 smithi179 ceph-mon[25623]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:03 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:03.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:11:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:11:03] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-05-25T06:11:03.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:04.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:03 smithi175 ceph-mon[24776]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:04.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:03 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:04.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:03 smithi175 ceph-mon[29436]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:04.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:03 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:05 smithi179 ceph-mon[25623]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:05.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:05 smithi175 ceph-mon[24776]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:05 smithi175 ceph-mon[29436]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:07.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:07 smithi179 ceph-mon[25623]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:07 smithi175 ceph-mon[24776]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:07 smithi175 ceph-mon[29436]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:09.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:09 smithi179 ceph-mon[25623]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:10.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:09 smithi175 ceph-mon[24776]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:09 smithi175 ceph-mon[29436]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:11 smithi179 ceph-mon[25623]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:11 smithi175 ceph-mon[24776]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:11 smithi175 ceph-mon[29436]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:13.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:13 smithi179 ceph-mon[25623]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:13.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:13 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:13.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:11:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:11:13] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-25T06:11:13.787 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:14.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:13 smithi175 ceph-mon[24776]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:14.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:13 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:14.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:13 smithi175 ceph-mon[29436]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:14.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:13 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:15 smithi179 ceph-mon[25623]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:15 smithi175 ceph-mon[24776]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:15 smithi175 ceph-mon[29436]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:17 smithi179 ceph-mon[25623]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:18.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:17 smithi175 ceph-mon[24776]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:17 smithi175 ceph-mon[29436]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:19.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:19 smithi179 ceph-mon[25623]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:20.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:19 smithi175 ceph-mon[24776]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:20.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:19 smithi175 ceph-mon[29436]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:20.971 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:20 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:20.971 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:20 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:20 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:21 smithi179 ceph-mon[25623]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:22.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:21 smithi175 ceph-mon[29436]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:22.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:21 smithi175 ceph-mon[24776]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:23 smithi179 ceph-mon[25623]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:23 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:23.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:11:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:11:23] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-25T06:11:23.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:24.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:23 smithi175 ceph-mon[24776]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:23 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:23 smithi175 ceph-mon[29436]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:23 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:25.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:25 smithi179 ceph-mon[25623]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:25.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:25 smithi175 ceph-mon[29436]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:25.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:25 smithi175 ceph-mon[24776]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:27 smithi179 ceph-mon[25623]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:28.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:27 smithi175 ceph-mon[24776]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:28.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:27 smithi175 ceph-mon[29436]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:29.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:30.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:29 smithi179 ceph-mon[25623]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:30.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:29 smithi175 ceph-mon[24776]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:30.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:29 smithi175 ceph-mon[29436]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:31 smithi179 ceph-mon[25623]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:32.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:31 smithi175 ceph-mon[24776]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:31 smithi175 ceph-mon[29436]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:33 smithi179 ceph-mon[25623]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:33 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:11:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:11:33] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-25T06:11:33.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:34.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:33 smithi175 ceph-mon[24776]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:33 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:34.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:33 smithi175 ceph-mon[29436]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:34.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:33 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:35.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:35 smithi179 ceph-mon[25623]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:35.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:35 smithi175 ceph-mon[24776]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:35 smithi175 ceph-mon[29436]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:37 smithi175 ceph-mon[24776]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:37 smithi175 ceph-mon[29436]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:37 smithi179 ceph-mon[25623]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:39.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:39 smithi179 ceph-mon[25623]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:40.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:39 smithi175 ceph-mon[24776]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:39 smithi175 ceph-mon[29436]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:41 smithi179 ceph-mon[25623]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:41 smithi175 ceph-mon[24776]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:41 smithi175 ceph-mon[29436]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:43.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:11:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:11:43] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-05-25T06:11:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:43 smithi179 ceph-mon[25623]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:43 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:43.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:44.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:43 smithi175 ceph-mon[24776]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:44.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:43 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:43 smithi175 ceph-mon[29436]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:43 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:45.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:45 smithi175 ceph-mon[24776]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:45 smithi175 ceph-mon[29436]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:45 smithi179 ceph-mon[25623]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:47 smithi179 ceph-mon[25623]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:48.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:47 smithi175 ceph-mon[29436]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:48.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:47 smithi175 ceph-mon[24776]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:48 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:11:49.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:49.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:48 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:11:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:48 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:11:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:49 smithi179 ceph-mon[25623]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:50.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:49 smithi175 ceph-mon[24776]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:50.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:49 smithi175 ceph-mon[29436]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:50.682 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:50 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:50.932 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:50 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:50 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:11:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:51 smithi179 ceph-mon[25623]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:52.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:51 smithi175 ceph-mon[29436]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:51 smithi175 ceph-mon[24776]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:53.414 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:11:53.415 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:53 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.504 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:53 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.504 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:53 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.504 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:53 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:11:53.504 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:53 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:11:53.504 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:53 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:11:53.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:11:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:11:53] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-25T06:11:53.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:11:54.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:54 smithi179 ceph-mon[25623]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:54.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:54 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:54.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:54 smithi175 ceph-mon[24776]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:54.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:54 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:54.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:54 smithi175 ceph-mon[29436]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:54.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:54 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:11:55.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:55 smithi175 ceph-mon[24776]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:55.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:55 smithi175 ceph-mon[29436]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:55 smithi179 ceph-mon[25623]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:11:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:57 smithi179 ceph-mon[25623]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:58.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:57 smithi175 ceph-mon[24776]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:57 smithi175 ceph-mon[29436]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:11:59.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:11:58 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:11:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:11:59 smithi179 ceph-mon[25623]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:00.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:11:59 smithi175 ceph-mon[24776]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:11:59 smithi175 ceph-mon[29436]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:01 smithi179 ceph-mon[25623]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:02.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:01 smithi175 ceph-mon[29436]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:01 smithi175 ceph-mon[24776]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:03 smithi179 ceph-mon[25623]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:03 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:03.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:12:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:12:03] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-25T06:12:03.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:03.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:03 smithi175 ceph-mon[24776]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:03 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:03.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:03 smithi175 ceph-mon[29436]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:03.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:03 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:05.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:05 smithi175 ceph-mon[24776]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:05 smithi175 ceph-mon[29436]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:05 smithi179 ceph-mon[25623]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:06.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:06.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:07.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:07 smithi175 ceph-mon[24776]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:07 smithi175 ceph-mon[29436]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:07 smithi179 ceph-mon[25623]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:09.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:09 smithi179 ceph-mon[25623]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:10.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:09 smithi175 ceph-mon[29436]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:09 smithi175 ceph-mon[24776]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:11 smithi179 ceph-mon[25623]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:11 smithi175 ceph-mon[29436]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:11 smithi175 ceph-mon[24776]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:13 smithi179 ceph-mon[25623]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:13 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:13.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:12:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:12:13] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-05-25T06:12:13.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:13.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:13 smithi175 ceph-mon[29436]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:13 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:13 smithi175 ceph-mon[24776]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:13 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:15.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:15 smithi175 ceph-mon[24776]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:15 smithi175 ceph-mon[29436]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:15 smithi179 ceph-mon[25623]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:17 smithi179 ceph-mon[25623]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:18.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:17 smithi175 ceph-mon[29436]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:17 smithi175 ceph-mon[24776]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:19.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:19 smithi179 ceph-mon[25623]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:20.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:19 smithi175 ceph-mon[24776]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:20.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:19 smithi175 ceph-mon[29436]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:20.971 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:20 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:20.971 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:20 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:20 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:21 smithi179 ceph-mon[25623]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:22.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:21 smithi175 ceph-mon[29436]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:22.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:21 smithi175 ceph-mon[24776]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:23 smithi179 ceph-mon[25623]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:23 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:23.786 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:12:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:12:23] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-05-25T06:12:23.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:23.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:23 smithi175 ceph-mon[29436]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:23 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:23 smithi175 ceph-mon[24776]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:23 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:25.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:25 smithi175 ceph-mon[24776]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:25 smithi175 ceph-mon[29436]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:25 smithi179 ceph-mon[25623]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:27 smithi179 ceph-mon[25623]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:28.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:27 smithi175 ceph-mon[24776]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:28.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:27 smithi175 ceph-mon[29436]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:29.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:30.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:29 smithi179 ceph-mon[25623]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:30.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:29 smithi175 ceph-mon[24776]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:30.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:29 smithi175 ceph-mon[29436]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:31 smithi179 ceph-mon[25623]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:32.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:31 smithi175 ceph-mon[24776]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:32.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:31 smithi175 ceph-mon[29436]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:33 smithi179 ceph-mon[25623]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:33 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:12:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:12:33] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-05-25T06:12:33.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi175\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi175\", job=\"node\", machine=\"x86_64\", nodename=\"smithi175\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:33.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:33 smithi175 ceph-mon[29436]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:33.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:33 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:33 smithi175 ceph-mon[24776]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:33 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:35.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:35 smithi175 ceph-mon[24776]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:35 smithi175 ceph-mon[29436]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:35 smithi179 ceph-mon[25623]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:36.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:36.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:37.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:37 smithi175 ceph-mon[24776]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:37 smithi175 ceph-mon[29436]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:37 smithi179 ceph-mon[25623]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:39.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"172.21.15.179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", cluster_addr=\"172.21.15.175\", device_class=\"ssd\", hostname=\"smithi175\", instance=\"smithi179:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.175\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:39 smithi179 ceph-mon[25623]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:40.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:39 smithi175 ceph-mon[24776]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:39 smithi175 ceph-mon[29436]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:41 smithi179 ceph-mon[25623]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:41 smithi175 ceph-mon[24776]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:41 smithi175 ceph-mon[29436]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:43 smithi179 ceph-mon[25623]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:43 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:43.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:12:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:12:43] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-05-25T06:12:43.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:12:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:12:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi179\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi179\", job=\"node\", machine=\"x86_64\", nodename=\"smithi179\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"0bfa7ec4-1a5b-11ef-bc9b-c7b262605968\", domainname=\"(none)\", instance=\"smithi179\", job=\"node\", machine=\"x86_64\", nodename=\"smithi179\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-25T06:12:43.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:43 smithi175 ceph-mon[29436]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:43 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:43 smithi175 ceph-mon[24776]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:43 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:45.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:45 smithi175 ceph-mon[29436]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:45 smithi175 ceph-mon[24776]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:45 smithi179 ceph-mon[25623]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:47 smithi179 ceph-mon[25623]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:48.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:47 smithi175 ceph-mon[24776]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:47 smithi175 ceph-mon[29436]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:49 smithi179 ceph-mon[25623]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:50.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:49 smithi175 ceph-mon[24776]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:50.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:49 smithi175 ceph-mon[29436]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:50.971 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:50 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:50.971 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:50 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:50 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:12:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:51 smithi179 ceph-mon[25623]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:52.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:51 smithi175 ceph-mon[24776]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:51 smithi175 ceph-mon[29436]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:53.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:12:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:12:53] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-05-25T06:12:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:53 smithi179 ceph-mon[25623]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:53 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:12:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:53 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:12:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:53 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:12:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:53 smithi179 ceph-mon[25623]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:12:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:53 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[24776]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:12:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:12:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:12:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[24776]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:12:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[29436]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:12:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:12:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:12:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[29436]: from='mgr.24788 ' entity='mgr.x' 2024-05-25T06:12:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:53 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:12:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:55 smithi175 ceph-mon[24776]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:55.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:55 smithi175 ceph-mon[29436]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:55 smithi179 ceph-mon[25623]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:12:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:57 smithi179 ceph-mon[25623]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:58.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:57 smithi175 ceph-mon[24776]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:12:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:57 smithi175 ceph-mon[29436]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:12:59 smithi179 ceph-mon[25623]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:00.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:12:59 smithi175 ceph-mon[24776]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:12:59 smithi175 ceph-mon[29436]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:01 smithi179 ceph-mon[25623]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:02.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:01 smithi175 ceph-mon[24776]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:01 smithi175 ceph-mon[29436]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:03 smithi179 ceph-mon[25623]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:03 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:03.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:13:03] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-05-25T06:13:03.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:03 smithi175 ceph-mon[24776]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:03.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:03 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:03.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:03 smithi175 ceph-mon[29436]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:03.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:03 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:05.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:05 smithi175 ceph-mon[24776]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:05 smithi175 ceph-mon[29436]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:05 smithi179 ceph-mon[25623]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:06 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:06 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:06.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:06 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:07 smithi175 ceph-mon[24776]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:07 smithi175 ceph-mon[29436]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:07 smithi179 ceph-mon[25623]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:09 smithi179 ceph-mon[25623]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:10.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:09 smithi175 ceph-mon[24776]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:09 smithi175 ceph-mon[29436]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:11 smithi179 ceph-mon[25623]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:11 smithi175 ceph-mon[24776]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:11 smithi175 ceph-mon[29436]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:13 smithi179 ceph-mon[25623]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:13 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:13.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:13:13] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-05-25T06:13:13.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:13 smithi175 ceph-mon[24776]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:13.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:13 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:13 smithi175 ceph-mon[29436]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:13 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:15.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:15 smithi175 ceph-mon[29436]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:15 smithi175 ceph-mon[24776]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:15 smithi179 ceph-mon[25623]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-25T06:13:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:17 smithi179 ceph-mon[25623]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:18.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:17 smithi175 ceph-mon[29436]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:17 smithi175 ceph-mon[24776]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:19 smithi179 ceph-mon[25623]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:20.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:19 smithi175 ceph-mon[29436]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:19 smithi175 ceph-mon[24776]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:20.971 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:20 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:20.971 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:20 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:20 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:21 smithi179 ceph-mon[25623]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:22.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:21 smithi175 ceph-mon[24776]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:22.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:21 smithi175 ceph-mon[29436]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:23 smithi179 ceph-mon[25623]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:23 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:13:23] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-25T06:13:23.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:23 smithi175 ceph-mon[24776]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:23 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:23 smithi175 ceph-mon[29436]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:23 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:25.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:25 smithi175 ceph-mon[29436]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:25.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:25 smithi175 ceph-mon[24776]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:25 smithi179 ceph-mon[25623]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:27 smithi179 ceph-mon[25623]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:28.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:27 smithi175 ceph-mon[29436]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:27 smithi175 ceph-mon[24776]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:30.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:29 smithi179 ceph-mon[25623]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:30.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:29 smithi175 ceph-mon[24776]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:30.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:29 smithi175 ceph-mon[29436]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:31 smithi179 ceph-mon[25623]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:31 smithi175 ceph-mon[29436]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:31 smithi175 ceph-mon[24776]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:32.762 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:13:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:33 smithi179 ceph-mon[25623]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:33 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:13:33] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-25T06:13:34.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:33 smithi175 ceph-mon[24776]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:33 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:34.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:33 smithi175 ceph-mon[29436]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:34.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:33 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:34.777 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:13:34.777 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (5m) 102s ago 13m 24.7M - 0.23.0 ba2b418f427c 1893f9da103a 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (5m) 3m ago 13m 46.3M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (2m) 102s ago 12m 54.7M - 3.5 e1d6a67b021e 7d117ccd9ed4 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283 running (8m) 3m ago 18m 509M - 19.0.0-2843-gea0684c0 f9a9809cafd6 2c455cbdf457 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (2m) 102s ago 19m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (19m) 102s ago 19m 58.8M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (18m) 3m ago 18m 47.3M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (18m) 102s ago 18m 47.1M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (6m) 102s ago 14m 18.7M - 1.3.1 1dbe0e931976 3fd7592e3fcc 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (5m) 3m ago 14m 20.1M - 1.3.1 1dbe0e931976 3142b4d09fa0 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (17m) 102s ago 17m 55.3M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (17m) 102s ago 17m 56.7M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (17m) 102s ago 17m 51.5M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:13:34.778 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (16m) 102s ago 16m 52.6M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:13:34.779 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (16m) 3m ago 16m 51.7M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:13:34.779 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (16m) 3m ago 16m 50.3M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:13:34.779 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (15m) 3m ago 15m 51.3M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:13:34.779 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (15m) 3m ago 15m 53.4M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:13:34.779 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (5m) 3m ago 14m 72.1M - 2.33.4 514e6a882f6e b84f89714aa4 2024-05-25T06:13:34.779 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (13m) 102s ago 12m 73.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:13:34.779 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (12m) 3m ago 12m 75.7M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:13:35.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions' 2024-05-25T06:13:35.621 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:35 smithi175 ceph-mon[29436]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:35.622 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:35 smithi175 ceph-mon[29436]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:13:35.622 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:35 smithi175 ceph-mon[24776]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:35.622 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:35 smithi175 ceph-mon[24776]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:13:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:35 smithi179 ceph-mon[25623]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:35 smithi179 ceph-mon[25623]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:13:36.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:36 smithi175 ceph-mon[24776]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:36 smithi175 ceph-mon[29436]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:36.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:36 smithi179 ceph-mon[25623]: from='mgr.24788 172.21.15.179:0/2693548212' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:37.207 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:13:37.207 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:13:37.208 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:13:37.209 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:13:37.209 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:13:37.209 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:13:37.209 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:13:37.209 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:13:37.209 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:13:37.209 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:13:37.817 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:37 smithi175 ceph-mon[29436]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:37.817 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:37 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/4095014102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:13:37.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:37 smithi175 ceph-mon[24776]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:37.818 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:37 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/4095014102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:13:37.819 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:13:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:37 smithi179 ceph-mon[25623]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:37 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/4095014102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:13:39.820 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_OK 2024-05-25T06:13:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:39 smithi179 ceph-mon[25623]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:40.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:39 smithi175 ceph-mon[24776]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:39 smithi175 ceph-mon[29436]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:40.375 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph -s' 2024-05-25T06:13:41.004 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:40 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2456910362' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:13:41.004 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:40 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2456910362' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:13:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:40 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2456910362' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:13:41.804 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:41 smithi175 ceph-mon[24776]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:41.805 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:41 smithi175 ceph-mon[29436]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:41 smithi179 ceph-mon[25623]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: cluster: 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: id: 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: health: HEALTH_OK 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: services: 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: mgr: x(active, since 6m), standbys: y 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-05-25T06:13:42.226 INFO:teuthology.orchestra.run.smithi175.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: data: 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: pools: 6 pools, 161 pgs 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: objects: 241 objects, 585 KiB 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: pgs: 161 active+clean 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: io: 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-25T06:13:42.227 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:13:42.773 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:42 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3790756494' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:13:42.774 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:42 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3790756494' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:13:42.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph mgr fail' 2024-05-25T06:13:43.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:42 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3790756494' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:13:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:43 smithi179 ceph-mon[25623]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:43 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:43.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:13:43] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-25T06:13:44.031 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:43 smithi175 ceph-mon[24776]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:44.031 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:43 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:44.031 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:43 smithi175 ceph-mon[29436]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:13:44.032 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:43 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:45.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[24776]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/240861500' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:13:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:13:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[24776]: osdmap e105: 8 total, 8 up, 8 in 2024-05-25T06:13:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[29436]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/240861500' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:13:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:13:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:45 smithi175 ceph-mon[29436]: osdmap e105: 8 total, 8 up, 8 in 2024-05-25T06:13:45.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:13:45] ENGINE Bus STOPPING 2024-05-25T06:13:45.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:13:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:13:45.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:13:45] ENGINE Bus STOPPED 2024-05-25T06:13:45.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:13:45] ENGINE Bus STARTING 2024-05-25T06:13:45.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:13:45] ENGINE Serving on http://:::9283 2024-05-25T06:13:45.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:13:45] ENGINE Bus STARTED 2024-05-25T06:13:45.940 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:45 smithi179 ceph-mon[25623]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:13:45.940 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:45 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/240861500' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:13:45.940 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:45 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:13:45.940 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:45 smithi179 ceph-mon[25623]: osdmap e105: 8 total, 8 up, 8 in 2024-05-25T06:13:45.941 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:45.565+0000 7faa38f39700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-25T06:13:45.941 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ignoring --setuser ceph since I am not root 2024-05-25T06:13:45.941 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ignoring --setgroup ceph since I am not root 2024-05-25T06:13:45.941 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:45 smithi179 ceph-mgr[45727]: -- 172.21.15.179:0/286166472 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d6087385a0 con 0x55d60870cc00 2024-05-25T06:13:45.941 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:45.750+0000 7fdd09bb7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:13:46.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'sleep 180' 2024-05-25T06:13:46.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:45.939+0000 7fdd09bb7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:13:46.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:46.049+0000 7fdd09bb7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:13:46.753 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:46.378+0000 7fdd09bb7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:13:46.753 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:46.473+0000 7fdd09bb7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: mgrmap e27: y(active, starting, since 0.765318s) 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:13:46.754 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: Manager daemon y is now available 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:13:46.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: mgrmap e27: y(active, starting, since 0.765318s) 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:13:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: Manager daemon y is now available 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: mgrmap e27: y(active, starting, since 0.765318s) 2024-05-25T06:13:46.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:13:46.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: Manager daemon y is now available 2024-05-25T06:13:46.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:13:46.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:13:46.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:13:46.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:13:47.736 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:47 smithi179 ceph-mon[25623]: [25/May/2024:06:13:45] ENGINE Bus STARTING 2024-05-25T06:13:47.736 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:47 smithi179 ceph-mon[25623]: [25/May/2024:06:13:46] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:13:47.736 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:47 smithi179 ceph-mon[25623]: [25/May/2024:06:13:46] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:13:47.736 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:47 smithi179 ceph-mon[25623]: [25/May/2024:06:13:46] ENGINE Bus STARTED 2024-05-25T06:13:47.736 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:47 smithi179 ceph-mon[25623]: mgrmap e28: y(active, since 1.77363s) 2024-05-25T06:13:47.736 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:47 smithi179 ceph-mon[25623]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:47.737 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:13:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:13:47.370Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.179:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.179:8765: connect: connection refused" 2024-05-25T06:13:47.737 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:13:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:13:47.370Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.179:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.179:8765: connect: connection refused" 2024-05-25T06:13:47.737 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:13:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:13:47.370Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.179:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.179:8765: connect: connection refused" 2024-05-25T06:13:47.737 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:13:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:13:47.372Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.179:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.179:8765: connect: connection refused" 2024-05-25T06:13:47.737 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:13:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:13:47.372Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.179:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.179:8765: connect: connection refused" 2024-05-25T06:13:47.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[29436]: [25/May/2024:06:13:45] ENGINE Bus STARTING 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[29436]: [25/May/2024:06:13:46] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[29436]: [25/May/2024:06:13:46] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[29436]: [25/May/2024:06:13:46] ENGINE Bus STARTED 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[29436]: mgrmap e28: y(active, since 1.77363s) 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[29436]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[24776]: [25/May/2024:06:13:45] ENGINE Bus STARTING 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[24776]: [25/May/2024:06:13:46] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:13:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[24776]: [25/May/2024:06:13:46] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:13:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[24776]: [25/May/2024:06:13:46] ENGINE Bus STARTED 2024-05-25T06:13:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[24776]: mgrmap e28: y(active, since 1.77363s) 2024-05-25T06:13:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:47 smithi175 ceph-mon[24776]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:48.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:47.735+0000 7fdd09bb7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:13:48.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:47.923+0000 7fdd09bb7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:13:48.427 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:48.122+0000 7fdd09bb7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:13:48.427 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:48.427+0000 7fdd09bb7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:13:48.743 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:48 smithi179 ceph-mon[25623]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:48.743 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:48.525+0000 7fdd09bb7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:13:48.866 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:48 smithi175 ceph-mon[29436]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:48.866 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:48 smithi175 ceph-mon[24776]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:49.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:48.742+0000 7fdd09bb7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:13:49.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:48.951+0000 7fdd09bb7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:13:49.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:49 smithi179 ceph-mon[25623]: mgrmap e29: y(active, since 3s) 2024-05-25T06:13:49.859 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:49.581+0000 7fdd09bb7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:13:49.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:49 smithi175 ceph-mon[29436]: mgrmap e29: y(active, since 3s) 2024-05-25T06:13:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:49 smithi175 ceph-mon[24776]: mgrmap e29: y(active, since 3s) 2024-05-25T06:13:50.125 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:50 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:50.025+0000 7fdd09bb7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:13:50.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:50 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:50.125+0000 7fdd09bb7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:13:50.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:50 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:50.345+0000 7fdd09bb7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:13:50.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:50 smithi175 ceph-mon[29436]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:50.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:50 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:50.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:50 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:50 smithi175 ceph-mon[24776]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:50 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:50 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:50.906 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:50 smithi179 ceph-mon[25623]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:50.906 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:50 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:50.906 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:50 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:51.264 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:50 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:50.906+0000 7fdd09bb7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:13:51.264 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:51.007+0000 7fdd09bb7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:13:51.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:51.263+0000 7fdd09bb7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:13:51.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:51.407+0000 7fdd09bb7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:13:51.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:51.506+0000 7fdd09bb7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:13:51.929 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:51.627+0000 7fdd09bb7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:13:51.929 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:51.929+0000 7fdd09bb7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:13:52.775 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:52 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.775 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:52 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.775 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:52 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.775 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:52 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.775 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:52 smithi179 ceph-mon[25623]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:52.776 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:52.539+0000 7fdd09bb7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:13:52.776 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:52.774+0000 7fdd09bb7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:13:52.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[29436]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:52 smithi175 ceph-mon[24776]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:13:53.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:13:52] ENGINE Bus STARTING 2024-05-25T06:13:53.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: CherryPy Checker: 2024-05-25T06:13:53.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: The Application mounted at '' has an empty config. 2024-05-25T06:13:53.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:13:53.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:13:52] ENGINE Serving on http://:::9283 2024-05-25T06:13:53.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:52 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:13:52] ENGINE Bus STARTED 2024-05-25T06:13:53.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:13:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:13:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: Standby manager daemon x started 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:13:53.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:53 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:13:53.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: Standby manager daemon x started 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:13:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:13:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:53 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:13:54.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: mgrmap e30: y(active, since 8s), standbys: x 2024-05-25T06:13:54.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:13:54.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:13:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:13:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:13:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:13:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:13:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:13:54.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: mgrmap e30: y(active, since 8s), standbys: x 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:13:54.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:54 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:13:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:13:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='client.15246 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: mgrmap e30: y(active, since 8s), standbys: x 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:13:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:13:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:54 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:13:55.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:55 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:13:55.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:55 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:13:55.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:55 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:13:55.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:55 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:13:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:55 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:13:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:55 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:13:56.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:56 smithi175 ceph-mon[24776]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:56 smithi175 ceph-mon[29436]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:13:56.562+0000 7fe2478c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:56.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:56.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:13:56.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:56.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:56.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:56.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:56.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:13:56 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:56 smithi179 ceph-mon[25623]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:57 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.622 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:57.623 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:13:57.624 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:57.625 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:57.626 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:57.627 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:57.627 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:57.627 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:57.627 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.627 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.627 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:57 smithi179 ceph-mon[25623]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:57 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:13:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:13:57.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:57 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:57.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:57.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:13:57.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:13:57.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:57 smithi175 ceph-mon[24776]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:13:57.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:13:57.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:13:57.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:13:57.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:13:57.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:13:57.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:57 smithi175 ceph-mon[29436]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:13:58.760 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:58 smithi179 ceph-mon[25623]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:13:58.760 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:13:58 smithi179 ceph-mon[25623]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-25T06:13:58.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:58 smithi175 ceph-mon[29436]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:13:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:13:58 smithi175 ceph-mon[29436]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-25T06:13:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:58 smithi175 ceph-mon[24776]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:13:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:13:58 smithi175 ceph-mon[24776]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-25T06:14:00.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 systemd[1]: Stopping Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:14:00.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 bash[54055]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:14:00.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.262Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:14:00.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[48801]: ts=2024-05-25T06:14:00.263Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-25T06:14:00.286 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 podman[54066]: 2024-05-25 06:14:00.273256982 +0000 UTC m=+0.093433245 container died b84f89714aa431a779540344d69987c8d33167e13db3d9f797bdbb2aea97d7cc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:14:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:00 smithi179 ceph-mon[25623]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:00.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 podman[54066]: 2024-05-25 06:14:00.596036601 +0000 UTC m=+0.416212859 container cleanup b84f89714aa431a779540344d69987c8d33167e13db3d9f797bdbb2aea97d7cc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:14:00.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 bash[54066]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a 2024-05-25T06:14:00.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 bash[54092]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:14:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:00 smithi175 ceph-mon[29436]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:00 smithi175 ceph-mon[24776]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:01.055 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 podman[54079]: 2024-05-25 06:14:00.788908246 +0000 UTC m=+0.517460634 container remove b84f89714aa431a779540344d69987c8d33167e13db3d9f797bdbb2aea97d7cc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:14:01.056 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-25T06:14:01.056 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 systemd[1]: Stopped Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:14:01.056 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Consumed 1.993s CPU time. 2024-05-25T06:14:01.056 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:00 smithi179 systemd[1]: Starting Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:14:01.056 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 podman[54157]: 2024-05-25 06:14:00.934525973 +0000 UTC m=+0.018458211 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-25T06:14:01.306 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 podman[54157]: 2024-05-25 06:14:01.056529835 +0000 UTC m=+0.140462057 container create 09ceb69012255099f6fdfa206948c9928c6f3578ef3564cff305757bc9990c1e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 podman[54157]: 2024-05-25 06:14:01.275555734 +0000 UTC m=+0.359487952 container init 09ceb69012255099f6fdfa206948c9928c6f3578ef3564cff305757bc9990c1e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 podman[54157]: 2024-05-25 06:14:01.27830063 +0000 UTC m=+0.362232856 container start 09ceb69012255099f6fdfa206948c9928c6f3578ef3564cff305757bc9990c1e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.301Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.301Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.301Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.301Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi179 (none))" 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.301Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.301Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.303Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.303Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.305Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-25T06:14:01.307 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.307Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-25T06:14:01.564 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.308Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=720.163µs 2024-05-25T06:14:01.565 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.308Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-25T06:14:01.565 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 bash[54157]: 09ceb69012255099f6fdfa206948c9928c6f3578ef3564cff305757bc9990c1e 2024-05-25T06:14:01.565 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 systemd[1]: Started Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:14:01.565 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.513Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-05-25T06:14:01.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:14:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:14:01.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.732Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-05-25T06:14:01.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:01.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:01.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:01.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:14:01.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.923Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.923Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.923Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.14µs wal_replay_duration=615.720151ms total_replay_duration=616.501664ms 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.928Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.928Z caller=main.go:947 level=info msg="TSDB started" 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.928Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.949Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.334475ms db_storage=886ns remote_storage=1.442µs web_handler=346ns query_engine=1.03µs scrape=426.206µs scrape_sd=98.396µs notify=18.098µs notify_sd=22.379µs rules=20.314564ms 2024-05-25T06:14:02.238 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:14:01 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:14:01.950Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-25T06:14:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:02 smithi179 ceph-mon[25623]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:14:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:02 smithi179 ceph-mon[25623]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:02 smithi175 ceph-mon[24776]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:14:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:02 smithi175 ceph-mon[24776]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:02 smithi175 ceph-mon[29436]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:14:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:02 smithi175 ceph-mon[29436]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:04 smithi179 ceph-mon[25623]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:05.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:04 smithi175 ceph-mon[24776]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:04 smithi175 ceph-mon[29436]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-25T06:14:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:14:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:14:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:14:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:14:06.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:06 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:14:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:14:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:14:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:14:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:14:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:14:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:06 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:14:07.703 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:07 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:14:07.704 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:07 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:14:07.704 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:07 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:14:07.704 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:07 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:14:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:07 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:14:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:07 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:14:08.727 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:08 smithi175 ceph-mon[29436]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:14:08.728 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:08 smithi175 ceph-mon[24776]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:14:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:08 smithi179 ceph-mon[25623]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-25T06:14:10.440 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:14:10.334+0000 7fe2478c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:10.440 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:10.440 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:10.440 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:14:10.441 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:10.442 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:10.442 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:10.442 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:10.442 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:10.442 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:10.442 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:14:10.443 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:14:10.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:10.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:10.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:10.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:10.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:10.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:10 smithi175 ceph-mon[24776]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:14:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:10 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/516271495' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:14:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:10 smithi175 ceph-mon[29436]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:14:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:10 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/516271495' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:14:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:10 smithi179 ceph-mon[25623]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:14:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:10 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/516271495' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:11 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:11.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:14:11.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:14:11.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:11.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:11 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:14:11.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:11.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:14:11.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:11.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:11.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:11.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:11.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:11 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:14:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:12.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:14:12.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi-foo-smithi175-buback 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: /usr/bin/podman: stderr Error: no such container ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-iscsi.foo.smithi175.buback 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:14:12.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:14:12.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:14:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:12 smithi179 ceph-mon[25623]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:14:13.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:12 smithi175 ceph-mon[24776]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:14:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:12 smithi175 ceph-mon[29436]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:14:13.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:14:13] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-05-25T06:14:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:14 smithi179 ceph-mon[25623]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:14:15.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:14 smithi175 ceph-mon[24776]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:14:15.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:14 smithi175 ceph-mon[29436]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:14:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:16 smithi179 ceph-mon[25623]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:14:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:16 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:17.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:16 smithi175 ceph-mon[24776]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:14:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:16 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:16 smithi175 ceph-mon[29436]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-25T06:14:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:16 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:18 smithi179 ceph-mon[25623]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:19.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:18 smithi175 ceph-mon[24776]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:18 smithi175 ceph-mon[29436]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:20.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:20 smithi175 ceph-mon[24776]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:20.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:20 smithi175 ceph-mon[29436]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:20 smithi179 ceph-mon[25623]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:21.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:22 smithi179 ceph-mon[25623]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:23.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:22 smithi175 ceph-mon[29436]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:23.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:22 smithi175 ceph-mon[24776]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-25T06:14:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:14:23] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-05-25T06:14:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:24 smithi179 ceph-mon[25623]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-25T06:14:25.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:24 smithi175 ceph-mon[24776]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-25T06:14:25.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:24 smithi175 ceph-mon[29436]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-25T06:14:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:26 smithi179 ceph-mon[25623]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:27.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:26 smithi175 ceph-mon[24776]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:27.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:26 smithi175 ceph-mon[29436]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:28 smithi179 ceph-mon[25623]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:29.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:28 smithi175 ceph-mon[29436]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:28 smithi175 ceph-mon[24776]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:30 smithi175 ceph-mon[24776]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:30 smithi175 ceph-mon[29436]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:30 smithi179 ceph-mon[25623]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:31 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:31 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:31 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:32 smithi179 ceph-mon[25623]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:33.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:32 smithi175 ceph-mon[24776]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:33.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:32 smithi175 ceph-mon[29436]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:33.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:14:33] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-05-25T06:14:35.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:34 smithi179 ceph-mon[25623]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:35.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:34 smithi175 ceph-mon[24776]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:35.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:34 smithi175 ceph-mon[29436]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:37.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:36 smithi179 ceph-mon[25623]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:37.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:36 smithi175 ceph-mon[29436]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:37.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:36 smithi175 ceph-mon[24776]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:38 smithi179 ceph-mon[25623]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:39.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:38 smithi175 ceph-mon[24776]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:38 smithi175 ceph-mon[29436]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:40.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:40 smithi175 ceph-mon[24776]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:40.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:40 smithi175 ceph-mon[29436]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:40 smithi179 ceph-mon[25623]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:42.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:43.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:42 smithi179 ceph-mon[25623]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:43.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:42 smithi175 ceph-mon[24776]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:43.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:42 smithi175 ceph-mon[29436]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:14:43] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-05-25T06:14:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:44 smithi179 ceph-mon[25623]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:45.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:44 smithi175 ceph-mon[24776]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:44 smithi175 ceph-mon[29436]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:46 smithi179 ceph-mon[25623]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:47.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:46 smithi175 ceph-mon[24776]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:46 smithi175 ceph-mon[29436]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:14:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:48 smithi179 ceph-mon[25623]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:49.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:48 smithi175 ceph-mon[24776]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:48 smithi175 ceph-mon[29436]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:50 smithi179 ceph-mon[25623]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:51.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:50 smithi175 ceph-mon[29436]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:50 smithi175 ceph-mon[24776]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:52.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:14:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:52 smithi179 ceph-mon[25623]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:53.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:52 smithi175 ceph-mon[29436]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:53.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:52 smithi175 ceph-mon[24776]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:53.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:14:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:14:53] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-05-25T06:14:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:54 smithi179 ceph-mon[25623]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:55.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:54 smithi175 ceph-mon[24776]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:55.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:54 smithi175 ceph-mon[29436]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:56 smithi179 ceph-mon[25623]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:57.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:56 smithi175 ceph-mon[24776]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:57.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:56 smithi175 ceph-mon[29436]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:14:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:14:58 smithi179 ceph-mon[25623]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:14:58 smithi175 ceph-mon[24776]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:14:59.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:14:58 smithi175 ceph-mon[29436]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:00 smithi179 ceph-mon[25623]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:01.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:00 smithi175 ceph-mon[24776]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:00 smithi175 ceph-mon[29436]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:02 smithi179 ceph-mon[25623]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:03.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:02 smithi175 ceph-mon[24776]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:02 smithi175 ceph-mon[29436]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:03.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:15:03] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.33.4" 2024-05-25T06:15:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:04 smithi179 ceph-mon[25623]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:05.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:04 smithi175 ceph-mon[24776]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:04 smithi175 ceph-mon[29436]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:06 smithi179 ceph-mon[25623]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:07.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:06 smithi175 ceph-mon[24776]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:07.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:06 smithi175 ceph-mon[29436]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:08 smithi179 ceph-mon[25623]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:08 smithi175 ceph-mon[29436]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:09.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:08 smithi175 ceph-mon[24776]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:10 smithi179 ceph-mon[25623]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:10 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:15:11.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:10 smithi175 ceph-mon[29436]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:11.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:10 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:15:11.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:10 smithi175 ceph-mon[24776]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:11.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:10 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:15:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:11 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:11 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:11 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:15:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:11 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:15:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:11 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:15:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:15:12.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:11 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:15:12.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[29436]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[24776]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:15:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:15:12.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:12 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:15:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:12 smithi179 ceph-mon[25623]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:12 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:15:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:12 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:15:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:12 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:15:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:12 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:15:13.240+0000 7fe2478c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:13.555 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:13.556 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:15:13.557 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:15:13.558 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:13.559 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:13.560 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:13.560 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:13.560 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:13.560 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:13.560 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:15:13] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.33.4" 2024-05-25T06:15:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:14 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:15:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:15:15.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:15.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:15.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:15.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:14 smithi179 ceph-mon[25623]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:14 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:15:15.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:15:15.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:15:15.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:14 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:15.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:15:15.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:15:15.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:14 smithi175 ceph-mon[29436]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:15.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:15:15.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:15:15.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:15:15.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:15:15.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:15:15.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:14 smithi175 ceph-mon[24776]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:16 smithi179 ceph-mon[25623]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:16 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:17.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:16 smithi175 ceph-mon[24776]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:16 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:16 smithi175 ceph-mon[29436]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:16 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:18 smithi179 ceph-mon[25623]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:19.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:18 smithi175 ceph-mon[24776]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:18 smithi175 ceph-mon[29436]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:20 smithi179 ceph-mon[25623]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:21.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:20 smithi175 ceph-mon[24776]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:20 smithi175 ceph-mon[29436]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:22.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:22.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:22 smithi175 ceph-mon[24776]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:22.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:22 smithi175 ceph-mon[29436]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:22 smithi179 ceph-mon[25623]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:23.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:15:23] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-05-25T06:15:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:24 smithi179 ceph-mon[25623]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:25.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:24 smithi175 ceph-mon[29436]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:25.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:24 smithi175 ceph-mon[24776]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:26 smithi179 ceph-mon[25623]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:27.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:26 smithi175 ceph-mon[24776]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:27.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:26 smithi175 ceph-mon[29436]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:28 smithi179 ceph-mon[25623]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:29.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:28 smithi175 ceph-mon[29436]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:28 smithi175 ceph-mon[24776]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:30 smithi179 ceph-mon[25623]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:31.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:30 smithi175 ceph-mon[24776]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:30 smithi175 ceph-mon[29436]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:31 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:32.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:31 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:31 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:32 smithi179 ceph-mon[25623]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:33.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:32 smithi175 ceph-mon[24776]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:32 smithi175 ceph-mon[29436]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:33.584 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:15:33] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-25T06:15:35.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:34 smithi179 ceph-mon[25623]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:35.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:34 smithi175 ceph-mon[29436]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:35.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:34 smithi175 ceph-mon[24776]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:37.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:36 smithi179 ceph-mon[25623]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:37.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:36 smithi175 ceph-mon[29436]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:37.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:36 smithi175 ceph-mon[24776]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:38 smithi179 ceph-mon[25623]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:39.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:38 smithi175 ceph-mon[29436]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:39.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:38 smithi175 ceph-mon[24776]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:40.998 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:40 smithi179 ceph-mon[25623]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:40 smithi175 ceph-mon[29436]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:40 smithi175 ceph-mon[24776]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:42.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:43.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:42 smithi179 ceph-mon[25623]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:43.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:42 smithi175 ceph-mon[29436]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:43.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:42 smithi175 ceph-mon[24776]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:15:43] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-25T06:15:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:44 smithi179 ceph-mon[25623]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:45.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:44 smithi175 ceph-mon[24776]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:44 smithi175 ceph-mon[29436]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:46 smithi179 ceph-mon[25623]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:46 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:47.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:46 smithi175 ceph-mon[24776]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:46 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:46 smithi175 ceph-mon[29436]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:46 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:15:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:48 smithi179 ceph-mon[25623]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:49.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:48 smithi175 ceph-mon[24776]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:48 smithi175 ceph-mon[29436]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:50 smithi179 ceph-mon[25623]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:51.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:50 smithi175 ceph-mon[24776]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:50 smithi175 ceph-mon[29436]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:52.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:15:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:52 smithi179 ceph-mon[25623]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:53.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:52 smithi175 ceph-mon[29436]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:53.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:52 smithi175 ceph-mon[24776]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:53.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:15:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:15:53] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-05-25T06:15:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:54 smithi179 ceph-mon[25623]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:55.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:54 smithi175 ceph-mon[29436]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:54 smithi175 ceph-mon[24776]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:56 smithi179 ceph-mon[25623]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:57.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:56 smithi175 ceph-mon[29436]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:57.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:56 smithi175 ceph-mon[24776]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:15:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:15:58 smithi179 ceph-mon[25623]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:59.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:15:58 smithi175 ceph-mon[29436]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:15:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:15:58 smithi175 ceph-mon[24776]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:00 smithi179 ceph-mon[25623]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:01.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:00 smithi175 ceph-mon[24776]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:00 smithi175 ceph-mon[29436]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:02.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:02 smithi179 ceph-mon[25623]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:03.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:02 smithi175 ceph-mon[24776]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:02 smithi175 ceph-mon[29436]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:16:03] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.33.4" 2024-05-25T06:16:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:04 smithi179 ceph-mon[25623]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:05.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:04 smithi175 ceph-mon[24776]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:04 smithi175 ceph-mon[29436]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:06 smithi179 ceph-mon[25623]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:07.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:06 smithi175 ceph-mon[24776]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:07.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:06 smithi175 ceph-mon[29436]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:08 smithi179 ceph-mon[25623]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:09.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:08 smithi175 ceph-mon[24776]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:08 smithi175 ceph-mon[29436]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:10 smithi179 ceph-mon[25623]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:11.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:10 smithi175 ceph-mon[24776]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:11.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:10 smithi175 ceph-mon[29436]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:12 smithi179 ceph-mon[25623]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:13.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:12 smithi175 ceph-mon[24776]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:12 smithi175 ceph-mon[29436]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:13.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:13 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:16:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:13 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:16:13.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:16:13] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.33.4" 2024-05-25T06:16:14.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:13 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:16:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:14 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:16:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:16:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:14 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:16:15.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:15 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:16:15.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:15 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:16:15.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:15 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:16:15.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:15 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:16:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:15 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:16:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:15 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:16:16.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:16:16.264+0000 7fe2478c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:16.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:16:16.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:16.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:16.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:16:16.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:16.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:16.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:16.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:16.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:16.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:16 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:16 smithi179 ceph-mon[25623]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:16 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:16 smithi175 ceph-mon[24776]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:16 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:16 smithi175 ceph-mon[29436]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:16 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:17 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:16:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:16:18.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:18.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:17 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:16:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:16:18.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:16:18.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:18.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:17 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:18.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:18.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:16:18.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:16:18.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:16:18.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:16:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:18 smithi179 ceph-mon[25623]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:19.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:18 smithi175 ceph-mon[24776]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:19.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:18 smithi175 ceph-mon[29436]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:20 smithi179 ceph-mon[25623]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:21.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:20 smithi175 ceph-mon[24776]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:21.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:20 smithi175 ceph-mon[29436]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:22.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:22.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:22 smithi179 ceph-mon[25623]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:23.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:22 smithi175 ceph-mon[24776]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:22 smithi175 ceph-mon[29436]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:16:23] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-05-25T06:16:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:24 smithi179 ceph-mon[25623]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:25.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:24 smithi175 ceph-mon[24776]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:25.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:24 smithi175 ceph-mon[29436]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:26 smithi179 ceph-mon[25623]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:27.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:26 smithi175 ceph-mon[24776]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:27.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:26 smithi175 ceph-mon[29436]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:28 smithi179 ceph-mon[25623]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:29.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:28 smithi175 ceph-mon[29436]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:28 smithi175 ceph-mon[24776]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:30 smithi179 ceph-mon[25623]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:31.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:30 smithi175 ceph-mon[24776]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:30 smithi175 ceph-mon[29436]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:31 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:32.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:31 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:31 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:32 smithi179 ceph-mon[25623]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:33.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:32 smithi175 ceph-mon[24776]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:32 smithi175 ceph-mon[29436]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:16:33] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-05-25T06:16:35.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:34 smithi179 ceph-mon[25623]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:35.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:34 smithi175 ceph-mon[24776]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:35.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:34 smithi175 ceph-mon[29436]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:37.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:36 smithi179 ceph-mon[25623]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:37.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:36 smithi175 ceph-mon[24776]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:36 smithi175 ceph-mon[29436]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:38 smithi179 ceph-mon[25623]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:39.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:38 smithi175 ceph-mon[29436]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:39.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:38 smithi175 ceph-mon[24776]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:40 smithi179 ceph-mon[25623]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:41.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:40 smithi175 ceph-mon[24776]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:40 smithi175 ceph-mon[29436]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:43.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:42 smithi179 ceph-mon[25623]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:43.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:42 smithi175 ceph-mon[24776]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:43.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:42 smithi175 ceph-mon[29436]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:16:43] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-05-25T06:16:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:44 smithi179 ceph-mon[25623]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:45.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:44 smithi175 ceph-mon[24776]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:44 smithi175 ceph-mon[29436]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:45 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:46.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:45 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:45 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:16:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:46 smithi179 ceph-mon[25623]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:47.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:46 smithi175 ceph-mon[29436]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:46 smithi175 ceph-mon[24776]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:48.528 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:16:48.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:48 smithi175 ceph-mon[24776]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:48.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:48 smithi175 ceph-mon[29436]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:48 smithi179 ceph-mon[25623]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:50.466 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (9m) 2m ago 17m 24.9M - 0.23.0 ba2b418f427c 1893f9da103a 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (8m) 2m ago 16m 55.4M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (6m) 2m ago 16m 59.4M - 3.5 e1d6a67b021e 7d117ccd9ed4 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283 running (11m) 2m ago 21m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 2c455cbdf457 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (6m) 2m ago 23m 477M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (23m) 2m ago 23m 59.2M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (21m) 2m ago 21m 49.8M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (21m) 2m ago 21m 46.2M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (9m) 2m ago 17m 21.4M - 1.3.1 1dbe0e931976 3fd7592e3fcc 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (9m) 2m ago 17m 17.1M - 1.3.1 1dbe0e931976 3142b4d09fa0 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (21m) 2m ago 21m 55.4M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:16:50.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (20m) 2m ago 20m 56.8M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (20m) 2m ago 20m 51.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (20m) 2m ago 20m 52.7M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (19m) 2m ago 19m 53.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (19m) 2m ago 19m 51.5M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (18m) 2m ago 18m 52.9M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (18m) 2m ago 18m 55.1M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (2m) 2m ago 17m 44.3M - 2.33.4 514e6a882f6e 09ceb6901225 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (16m) 2m ago 16m 73.7M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:16:50.468 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (16m) 2m ago 16m 77.2M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:16:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:50 smithi179 ceph-mon[25623]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:51.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:50 smithi175 ceph-mon[24776]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:50 smithi175 ceph-mon[29436]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:51.127 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions' 2024-05-25T06:16:51.706 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:51.706 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:51 smithi175 ceph-mon[24776]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:16:51.707 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:51.707 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:51 smithi175 ceph-mon[29436]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:16:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:16:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:51 smithi179 ceph-mon[25623]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:16:52.846 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:52 smithi175 ceph-mon[24776]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:52.847 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:52 smithi175 ceph-mon[29436]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:52 smithi179 ceph-mon[25623]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:16:53.237 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:16:53.237 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:16:53.237 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:16:53.237 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:16:53.238 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:16:53.239 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:16:53.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:53 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2508622308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:16:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:53 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2508622308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:16:53.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:16:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:16:53] "GET /metrics HTTP/1.1" 200 37918 "" "Prometheus/2.33.4" 2024-05-25T06:16:53.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph -s' 2024-05-25T06:16:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:53 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2508622308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:16:54.791 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:54 smithi175 ceph-mon[29436]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:54.791 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:54 smithi175 ceph-mon[24776]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:54 smithi179 ceph-mon[25623]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:55.831 INFO:teuthology.orchestra.run.smithi175.stdout: cluster: 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: id: 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: health: HEALTH_OK 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: services: 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: mgr: y(active, since 3m), standbys: x 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: data: 2024-05-25T06:16:55.832 INFO:teuthology.orchestra.run.smithi175.stdout: pools: 6 pools, 161 pgs 2024-05-25T06:16:55.833 INFO:teuthology.orchestra.run.smithi175.stdout: objects: 241 objects, 585 KiB 2024-05-25T06:16:55.833 INFO:teuthology.orchestra.run.smithi175.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:16:55.833 INFO:teuthology.orchestra.run.smithi175.stdout: pgs: 161 active+clean 2024-05-25T06:16:55.833 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:16:55.833 INFO:teuthology.orchestra.run.smithi175.stdout: io: 2024-05-25T06:16:55.833 INFO:teuthology.orchestra.run.smithi175.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-25T06:16:55.833 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-05-25T06:16:56.416 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:16:56.952 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:56 smithi175 ceph-mon[24776]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:56.952 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:56 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3261035509' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:16:56.952 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:56 smithi175 ceph-mon[29436]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:56.952 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:56 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3261035509' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:16:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:56 smithi179 ceph-mon[25623]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:16:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:56 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3261035509' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-25T06:16:58.505 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_OK 2024-05-25T06:16:58.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:58 smithi175 ceph-mon[24776]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:16:58 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/664225571' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:16:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:58 smithi175 ceph-mon[29436]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:16:58 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/664225571' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:16:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:58 smithi179 ceph-mon[25623]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:16:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:16:58 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/664225571' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:16:59.145 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-25T06:17:00.643 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:00 smithi175 ceph-mon[29436]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:00.644 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:00 smithi175 ceph-mon[24776]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:00 smithi179 ceph-mon[25623]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:01.171 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:17:01.751 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:01.751 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:01 smithi175 ceph-mon[24776]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:01.751 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:01 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/338271091' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:17:01.751 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:01.752 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:01 smithi175 ceph-mon[29436]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:01.752 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:01 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/338271091' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:17:01.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph mgr fail' 2024-05-25T06:17:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:01 smithi179 ceph-mon[25623]: from='mgr.15231 172.21.15.175:0/2382470922' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:01 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/338271091' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:17:02.724 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:02 smithi175 ceph-mon[29436]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:02.725 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:02 smithi175 ceph-mon[24776]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:02 smithi179 ceph-mon[25623]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:03.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:17:03] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-05-25T06:17:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:04 smithi179 ceph-mon[25623]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:04 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3686939248' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:17:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:04 smithi179 ceph-mon[25623]: osdmap e106: 8 total, 8 up, 8 in 2024-05-25T06:17:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:04 smithi179 ceph-mon[25623]: Standby manager daemon y started 2024-05-25T06:17:05.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:17:04] ENGINE Bus STOPPING 2024-05-25T06:17:05.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:17:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:17:05.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:17:04] ENGINE Bus STOPPED 2024-05-25T06:17:05.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:17:04] ENGINE Bus STARTING 2024-05-25T06:17:05.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:17:04] ENGINE Serving on http://:::9283 2024-05-25T06:17:05.036 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:17:04] ENGINE Bus STARTED 2024-05-25T06:17:05.066 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[29436]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:05.066 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3686939248' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:17:05.066 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[29436]: osdmap e106: 8 total, 8 up, 8 in 2024-05-25T06:17:05.066 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[29436]: Standby manager daemon y started 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[24776]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3686939248' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[24776]: osdmap e106: 8 total, 8 up, 8 in 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mon[24776]: Standby manager daemon y started 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ignoring --setuser ceph since I am not root 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ignoring --setgroup ceph since I am not root 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:04 smithi175 ceph-mgr[63288]: -- 172.21.15.175:0/4282926595 <== mon.0 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5629ed15a5a0 con 0x5629ed12e800 2024-05-25T06:17:05.067 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:04.886+0000 7f8040f5d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:17:05.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:05.080+0000 7f8040f5d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:17:05.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:05.190+0000 7f8040f5d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:17:05.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'sleep 180' 2024-05-25T06:17:05.635 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:05.529+0000 7f8040f5d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:17:05.973 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:17:05.973 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:17:05.973 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3686939248' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: mgrmap e31: x(active, starting, since 0.949712s), standbys: y 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:17:05.974 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: Manager daemon x is now available 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:17:05.975 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:05 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:17:05.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:05.634+0000 7f8040f5d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:17:05.978 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:17:05.978 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:17:05.978 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3686939248' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: mgrmap e31: x(active, starting, since 0.949712s), standbys: y 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:17:05.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: Manager daemon x is now available 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:17:05.980 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3686939248' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: mgrmap e31: x(active, starting, since 0.949712s), standbys: y 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:17:05.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:17:05.982 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: Manager daemon x is now available 2024-05-25T06:17:05.982 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:17:05.982 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:05.982 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:17:05.982 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:17:05.982 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:17:05.982 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:05 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-25T06:17:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:06 smithi179 ceph-mon[25623]: mgrmap e32: x(active, since 1.96939s), standbys: y 2024-05-25T06:17:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:06 smithi179 ceph-mon[25623]: [25/May/2024:06:17:05] ENGINE Bus STARTING 2024-05-25T06:17:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:06 smithi179 ceph-mon[25623]: [25/May/2024:06:17:05] ENGINE Serving on https://172.21.15.179:7150 2024-05-25T06:17:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:06 smithi179 ceph-mon[25623]: [25/May/2024:06:17:06] ENGINE Serving on http://172.21.15.179:8765 2024-05-25T06:17:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:06 smithi179 ceph-mon[25623]: [25/May/2024:06:17:06] ENGINE Bus STARTED 2024-05-25T06:17:06.998 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[29436]: mgrmap e32: x(active, since 1.96939s), standbys: y 2024-05-25T06:17:06.998 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[29436]: [25/May/2024:06:17:05] ENGINE Bus STARTING 2024-05-25T06:17:06.998 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[29436]: [25/May/2024:06:17:05] ENGINE Serving on https://172.21.15.179:7150 2024-05-25T06:17:06.998 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[29436]: [25/May/2024:06:17:06] ENGINE Serving on http://172.21.15.179:8765 2024-05-25T06:17:06.999 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[29436]: [25/May/2024:06:17:06] ENGINE Bus STARTED 2024-05-25T06:17:06.999 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:06.869+0000 7f8040f5d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:17:06.999 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[24776]: mgrmap e32: x(active, since 1.96939s), standbys: y 2024-05-25T06:17:06.999 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[24776]: [25/May/2024:06:17:05] ENGINE Bus STARTING 2024-05-25T06:17:06.999 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[24776]: [25/May/2024:06:17:05] ENGINE Serving on https://172.21.15.179:7150 2024-05-25T06:17:06.999 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[24776]: [25/May/2024:06:17:06] ENGINE Serving on http://172.21.15.179:8765 2024-05-25T06:17:06.999 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:06 smithi175 ceph-mon[24776]: [25/May/2024:06:17:06] ENGINE Bus STARTED 2024-05-25T06:17:07.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:07.052+0000 7f8040f5d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:17:07.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:07.226+0000 7f8040f5d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:17:07.727 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:07.536+0000 7f8040f5d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:17:07.727 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:07.632+0000 7f8040f5d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:17:08.025 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:07.851+0000 7f8040f5d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:17:08.026 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:07 smithi175 ceph-mon[29436]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:08.026 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:07 smithi175 ceph-mon[24776]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:07 smithi179 ceph-mon[25623]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:08.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:08 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:08.052+0000 7f8040f5d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:17:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:08 smithi179 ceph-mon[25623]: mgrmap e33: x(active, since 3s), standbys: y 2024-05-25T06:17:09.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:08 smithi175 ceph-mon[29436]: mgrmap e33: x(active, since 3s), standbys: y 2024-05-25T06:17:09.051 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:08 smithi175 ceph-mon[24776]: mgrmap e33: x(active, since 3s), standbys: y 2024-05-25T06:17:09.052 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:08 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:08.737+0000 7f8040f5d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:17:09.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:09.192+0000 7f8040f5d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:17:09.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:09.294+0000 7f8040f5d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:17:09.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:09.526+0000 7f8040f5d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:17:10.130 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:10.068+0000 7f8040f5d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:17:10.384 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:10 smithi175 ceph-mon[29436]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:10.384 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:10 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:10.384 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:10 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:10.385 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:10.185+0000 7f8040f5d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:17:10.385 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:10 smithi175 ceph-mon[24776]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:10.385 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:10 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:10.385 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:10 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:10 smithi179 ceph-mon[25623]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:10 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:10 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:10.691 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:10.447+0000 7f8040f5d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:17:10.691 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:10.591+0000 7f8040f5d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:17:10.691 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:10.691+0000 7f8040f5d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:17:11.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:10.811+0000 7f8040f5d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:17:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:11 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:11 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:11 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:11 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:11 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:11.136+0000 7f8040f5d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:11.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:11 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:12.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:11 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:11.788+0000 7f8040f5d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:17:12.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:12 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:12.025+0000 7f8040f5d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:17:12.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:12 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:17:12] ENGINE Bus STARTING 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: Standby manager daemon y restarted 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: Standby manager daemon y started 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:17:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:17:12.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:12.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:17:12.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:12 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:17:12.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:12.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: Standby manager daemon y restarted 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: Standby manager daemon y started 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:17:12.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:12 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CherryPy Checker: 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:12 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: The Application mounted at '' has an empty config. 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:12 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:12 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:17:12] ENGINE Serving on http://:::9283 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:12 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:17:12] ENGINE Bus STARTED 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: Standby manager daemon y restarted 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: Standby manager daemon y started 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:12.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-25T06:17:12.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:17:12.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-25T06:17:12.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:17:12.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:12.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:17:12.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:12 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:17:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: mgrmap e34: x(active, since 8s), standbys: y 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:13 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: mgrmap e34: x(active, since 8s), standbys: y 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: mgrmap e34: x(active, since 8s), standbys: y 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:17:13.557 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.558 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.558 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.558 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.558 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:13 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:13.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:17:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:17:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:17:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:14 smithi179 ceph-mon[25623]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:17:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:14 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:17:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:14 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:17:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:14 smithi179 ceph-mon[25623]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-05-25T06:17:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:14 smithi179 ceph-mon[25623]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:17:14.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:14 smithi179 ceph-mon[25623]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:17:14.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:14 smithi179 ceph-mon[25623]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:17:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[29436]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:17:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:17:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:17:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[29436]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-05-25T06:17:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[29436]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:17:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[29436]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:17:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[29436]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:17:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[24776]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:17:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:17:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:17:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[24776]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-05-25T06:17:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[24776]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:17:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[24776]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:17:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:14 smithi175 ceph-mon[24776]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:17:16.699 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:16 smithi179 ceph-mon[25623]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:16.699 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 systemd[1]: Stopping Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:17:16.699 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 bash[57076]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:17:16.958 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:17:16.958 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.699Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.700Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.700Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[54171]: ts=2024-05-25T06:17:16.700Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-25T06:17:16.959 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:16 smithi179 podman[57087]: 2024-05-25 06:17:16.709333315 +0000 UTC m=+0.141219701 container died 09ceb69012255099f6fdfa206948c9928c6f3578ef3564cff305757bc9990c1e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:17:17.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:16 smithi175 ceph-mon[24776]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:16 smithi175 ceph-mon[29436]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:17.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 podman[57087]: 2024-05-25 06:17:17.057388108 +0000 UTC m=+0.489274548 container cleanup 09ceb69012255099f6fdfa206948c9928c6f3578ef3564cff305757bc9990c1e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:17:17.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 bash[57087]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a 2024-05-25T06:17:17.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 bash[57113]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:17:17.676 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 podman[57098]: 2024-05-25 06:17:17.341842666 +0000 UTC m=+0.633752904 container remove 09ceb69012255099f6fdfa206948c9928c6f3578ef3564cff305757bc9990c1e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:17:17.676 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-25T06:17:17.676 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 systemd[1]: Stopped Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:17:17.676 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Consumed 1.379s CPU time. 2024-05-25T06:17:17.676 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 systemd[1]: Starting Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:17:17.677 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 podman[57178]: 2024-05-25 06:17:17.575958582 +0000 UTC m=+0.105758900 container create 35dc48baab92354e0038228ba53872195b343be75a3e9530db8c79b47ad5def0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:17:17.677 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 podman[57178]: 2024-05-25 06:17:17.488256823 +0000 UTC m=+0.018057154 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 podman[57178]: 2024-05-25 06:17:17.803389639 +0000 UTC m=+0.333189956 container init 35dc48baab92354e0038228ba53872195b343be75a3e9530db8c79b47ad5def0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 podman[57178]: 2024-05-25 06:17:17.806409464 +0000 UTC m=+0.336209792 container start 35dc48baab92354e0038228ba53872195b343be75a3e9530db8c79b47ad5def0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.830Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.830Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.830Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.830Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi179 (none))" 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.830Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.830Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.832Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.832Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-25T06:17:18.038 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.833Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-25T06:17:18.039 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.836Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-25T06:17:18.039 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.836Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=653.067µs 2024-05-25T06:17:18.039 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:17.836Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-25T06:17:18.039 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 bash[57178]: 35dc48baab92354e0038228ba53872195b343be75a3e9530db8c79b47ad5def0 2024-05-25T06:17:18.039 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:17 smithi179 systemd[1]: Started Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:17:18.485 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.041Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-05-25T06:17:18.485 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.264Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-05-25T06:17:18.486 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.484Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-05-25T06:17:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:18 smithi179 ceph-mon[25623]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:18 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:18 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:18 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:17:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:18 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:17:18.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.539Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-05-25T06:17:18.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.539Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-05-25T06:17:18.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.539Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.763µs wal_replay_duration=702.579445ms total_replay_duration=703.28863ms 2024-05-25T06:17:18.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.546Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-25T06:17:18.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.546Z caller=main.go:947 level=info msg="TSDB started" 2024-05-25T06:17:18.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.546Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-25T06:17:18.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.567Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.284419ms db_storage=952ns remote_storage=1.764µs web_handler=428ns query_engine=1.095µs scrape=436.831µs scrape_sd=104.961µs notify=11.22µs notify_sd=14.055µs rules=20.061331ms 2024-05-25T06:17:18.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:17:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:17:18.567Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-25T06:17:19.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[29436]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[24776]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:17:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:18 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:17:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:19 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:17:19.983 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:19 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:17:19.984 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:19 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:17:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:20 smithi179 ceph-mon[25623]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:20 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:20 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:20 smithi175 ceph-mon[29436]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:20 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:20 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:20 smithi175 ceph-mon[24776]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:20 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:20 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:21.911 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:22.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:22.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:22 smithi179 ceph-mon[25623]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:22 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:22 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:22 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:17:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:22 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:17:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:22 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[24776]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[29436]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:23.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:17:23.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:17:23.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:22 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:17:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:23 smithi179 ceph-mon[25623]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:23 smithi179 ceph-mon[25623]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:23 smithi175 ceph-mon[24776]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:23 smithi175 ceph-mon[24776]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:23 smithi175 ceph-mon[29436]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:23 smithi175 ceph-mon[29436]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:25 smithi179 ceph-mon[25623]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:26.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:25 smithi175 ceph-mon[24776]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:26.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:25 smithi175 ceph-mon[29436]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:27 smithi179 ceph-mon[25623]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:28.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:27 smithi175 ceph-mon[24776]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:28.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:27 smithi175 ceph-mon[29436]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:29.994 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:29 smithi175 ceph-mon[24776]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:29.995 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:29 smithi175 ceph-mon[29436]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:30.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:29 smithi179 ceph-mon[25623]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:17:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:31 smithi179 ceph-mon[25623]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 564 B/s rd, 0 op/s 2024-05-25T06:17:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:31 smithi175 ceph-mon[24776]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 564 B/s rd, 0 op/s 2024-05-25T06:17:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:31 smithi175 ceph-mon[29436]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 564 B/s rd, 0 op/s 2024-05-25T06:17:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:33 smithi179 ceph-mon[25623]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:17:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:17:33] "GET /metrics HTTP/1.1" 200 37998 "" "Prometheus/2.33.4" 2024-05-25T06:17:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:33 smithi175 ceph-mon[24776]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:17:34.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:33 smithi175 ceph-mon[29436]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:17:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:35 smithi179 ceph-mon[25623]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:35 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:36.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:35 smithi175 ceph-mon[24776]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:35 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:35 smithi175 ceph-mon[29436]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:35 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:37 smithi179 ceph-mon[25623]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:38.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:37 smithi175 ceph-mon[24776]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:38.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:37 smithi175 ceph-mon[29436]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:39.996 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:39 smithi175 ceph-mon[24776]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:39.996 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:39 smithi175 ceph-mon[29436]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:39 smithi179 ceph-mon[25623]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:41 smithi179 ceph-mon[25623]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:41 smithi175 ceph-mon[24776]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:41 smithi175 ceph-mon[29436]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:43 smithi179 ceph-mon[25623]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:43.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:17:43] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.33.4" 2024-05-25T06:17:44.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:43 smithi175 ceph-mon[24776]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:43 smithi175 ceph-mon[29436]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:45 smithi179 ceph-mon[25623]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:46.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:45 smithi175 ceph-mon[24776]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:45 smithi175 ceph-mon[29436]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:46 smithi179 ceph-mon[25623]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:47.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:46 smithi175 ceph-mon[24776]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:46 smithi175 ceph-mon[29436]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:49 smithi179 ceph-mon[25623]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:49.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:49 smithi175 ceph-mon[24776]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:49 smithi175 ceph-mon[29436]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:50 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:51.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:50 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:50 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:17:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:51 smithi179 ceph-mon[25623]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:52.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:52.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:51 smithi175 ceph-mon[29436]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:17:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:51 smithi175 ceph-mon[24776]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:53 smithi179 ceph-mon[25623]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:53.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:17:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:17:53] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-05-25T06:17:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:53 smithi175 ceph-mon[24776]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:53 smithi175 ceph-mon[29436]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:55 smithi179 ceph-mon[25623]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:56.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:55 smithi175 ceph-mon[24776]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:56.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:55 smithi175 ceph-mon[29436]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:17:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:57 smithi179 ceph-mon[25623]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:58.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:57 smithi175 ceph-mon[24776]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:17:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:57 smithi175 ceph-mon[29436]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:00.018 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:17:59 smithi175 ceph-mon[24776]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:00.019 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:17:59 smithi175 ceph-mon[29436]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:17:59 smithi179 ceph-mon[25623]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:01 smithi179 ceph-mon[25623]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:01 smithi175 ceph-mon[24776]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:01 smithi175 ceph-mon[29436]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:03 smithi179 ceph-mon[25623]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:03.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:18:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:18:03] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-05-25T06:18:04.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:03 smithi175 ceph-mon[24776]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:03 smithi175 ceph-mon[29436]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:05 smithi179 ceph-mon[25623]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:06.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:05 smithi175 ceph-mon[24776]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:06.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:05 smithi175 ceph-mon[29436]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:07 smithi179 ceph-mon[25623]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:07 smithi175 ceph-mon[24776]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:08.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:07 smithi175 ceph-mon[29436]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:10.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:09 smithi175 ceph-mon[24776]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:10.027 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:09 smithi175 ceph-mon[29436]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:09 smithi179 ceph-mon[25623]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:11 smithi179 ceph-mon[25623]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:11 smithi175 ceph-mon[24776]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:11 smithi175 ceph-mon[29436]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:13 smithi179 ceph-mon[25623]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:13.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:18:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:18:13] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-05-25T06:18:14.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:13 smithi175 ceph-mon[24776]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:14.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:13 smithi175 ceph-mon[29436]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:15 smithi179 ceph-mon[25623]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:16.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:15 smithi175 ceph-mon[24776]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:15 smithi175 ceph-mon[29436]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:17 smithi179 ceph-mon[25623]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:18.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:17 smithi175 ceph-mon[24776]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:17 smithi175 ceph-mon[29436]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:20.033 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:19 smithi175 ceph-mon[24776]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:20.033 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:19 smithi175 ceph-mon[29436]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:20.034 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:19 smithi179 ceph-mon[25623]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:20 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:21.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:20 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:20 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:21 smithi179 ceph-mon[25623]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:22.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:22.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:21 smithi175 ceph-mon[29436]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:22.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:22.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:21 smithi175 ceph-mon[24776]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:22 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:18:23.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:22 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:18:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:22 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:18:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:18:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:18:23] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-05-25T06:18:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:23 smithi179 ceph-mon[25623]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:23 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:18:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:23 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:18:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:23 smithi179 ceph-mon[25623]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-25T06:18:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:23 smithi179 ceph-mon[25623]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:23 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:18:24.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[24776]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[24776]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[24776]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[29436]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:18:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[29436]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-25T06:18:24.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[29436]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:24.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:23 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:18:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:26 smithi179 ceph-mon[25623]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:18:27.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:26 smithi175 ceph-mon[24776]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:18:27.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:26 smithi175 ceph-mon[29436]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:18:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:28 smithi179 ceph-mon[25623]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:29.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:28 smithi175 ceph-mon[29436]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:28 smithi175 ceph-mon[24776]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:30 smithi179 ceph-mon[25623]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:31.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:30 smithi175 ceph-mon[24776]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:30 smithi175 ceph-mon[29436]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:32.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:32.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:32 smithi179 ceph-mon[25623]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:33.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:32 smithi175 ceph-mon[24776]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:32 smithi175 ceph-mon[29436]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:18:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:18:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:18:33] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-05-25T06:18:35.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:34 smithi179 ceph-mon[25623]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:18:35.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:34 smithi175 ceph-mon[24776]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:18:35.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:34 smithi175 ceph-mon[29436]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:18:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:35 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:36.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:35 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:35 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:37.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:36 smithi179 ceph-mon[25623]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:37.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:36 smithi175 ceph-mon[24776]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:36 smithi175 ceph-mon[29436]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:38 smithi179 ceph-mon[25623]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:39.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:38 smithi175 ceph-mon[24776]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:39.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:38 smithi175 ceph-mon[29436]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:40 smithi179 ceph-mon[25623]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:41.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:40 smithi175 ceph-mon[29436]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:40 smithi175 ceph-mon[24776]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:42.013 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:43.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:42 smithi179 ceph-mon[25623]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:43.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:42 smithi175 ceph-mon[24776]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:43.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:42 smithi175 ceph-mon[29436]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:43.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:18:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:18:43] "GET /metrics HTTP/1.1" 200 37995 "" "Prometheus/2.33.4" 2024-05-25T06:18:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:44 smithi179 ceph-mon[25623]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:45.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:44 smithi175 ceph-mon[24776]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:45.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:44 smithi175 ceph-mon[29436]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:45 smithi179 ceph-mon[25623]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:46.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:45 smithi175 ceph-mon[24776]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:45 smithi175 ceph-mon[29436]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:48 smithi179 ceph-mon[25623]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:48.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:48 smithi175 ceph-mon[24776]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:48.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:48 smithi175 ceph-mon[29436]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:50 smithi179 ceph-mon[25623]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:50 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:51.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:50 smithi175 ceph-mon[29436]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:50 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:50 smithi175 ceph-mon[24776]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:50 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:18:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:52.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:18:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:52 smithi179 ceph-mon[25623]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:53.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:52 smithi175 ceph-mon[24776]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:53.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:52 smithi175 ceph-mon[29436]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:53.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:18:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:18:53] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-05-25T06:18:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:54 smithi179 ceph-mon[25623]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:55.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:54 smithi175 ceph-mon[24776]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:55.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:54 smithi175 ceph-mon[29436]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:56 smithi179 ceph-mon[25623]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:57.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:56 smithi175 ceph-mon[29436]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:57.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:56 smithi175 ceph-mon[24776]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:18:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:18:58 smithi179 ceph-mon[25623]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:59.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:18:58 smithi175 ceph-mon[24776]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:18:59.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:18:58 smithi175 ceph-mon[29436]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:00 smithi179 ceph-mon[25623]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:01.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:00 smithi175 ceph-mon[29436]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:01.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:00 smithi175 ceph-mon[24776]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:02.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:02 smithi179 ceph-mon[25623]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:03.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:02 smithi175 ceph-mon[24776]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:02 smithi175 ceph-mon[29436]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:03.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:19:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:19:03] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-05-25T06:19:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:04 smithi179 ceph-mon[25623]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:05.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:04 smithi175 ceph-mon[24776]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:04 smithi175 ceph-mon[29436]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:06.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:06.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:06 smithi179 ceph-mon[25623]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:07.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:06 smithi175 ceph-mon[29436]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:07.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:06 smithi175 ceph-mon[24776]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:08 smithi179 ceph-mon[25623]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:09.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:08 smithi175 ceph-mon[24776]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:08 smithi175 ceph-mon[29436]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:10 smithi179 ceph-mon[25623]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:11.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:10 smithi175 ceph-mon[29436]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:11.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:10 smithi175 ceph-mon[24776]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:12 smithi179 ceph-mon[25623]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:13.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:12 smithi175 ceph-mon[29436]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:12 smithi175 ceph-mon[24776]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:13.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:19:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:19:13] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-05-25T06:19:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:14 smithi179 ceph-mon[25623]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:15.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:14 smithi175 ceph-mon[29436]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:14 smithi175 ceph-mon[24776]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:16 smithi179 ceph-mon[25623]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:16 smithi175 ceph-mon[29436]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:16 smithi175 ceph-mon[24776]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:18 smithi179 ceph-mon[25623]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:19.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:18 smithi175 ceph-mon[29436]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:18 smithi175 ceph-mon[24776]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:20 smithi179 ceph-mon[25623]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:20 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:21.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:20 smithi175 ceph-mon[29436]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:21.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:20 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:20 smithi175 ceph-mon[24776]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:20 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:22.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:22.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:22 smithi179 ceph-mon[25623]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:23.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:22 smithi175 ceph-mon[24776]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:22 smithi175 ceph-mon[29436]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:23 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:19:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:19:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:19:23] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.33.4" 2024-05-25T06:19:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:23 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:19:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:23 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:19:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:24 smithi179 ceph-mon[25623]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:24 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:19:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:24 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:19:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:24 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:19:25.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[29436]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:25.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:19:25.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:19:25.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:19:25.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[24776]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:25.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:19:25.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:19:25.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:24 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:19:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:25 smithi179 ceph-mon[25623]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-25T06:19:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:25 smithi179 ceph-mon[25623]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:26.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:25 smithi175 ceph-mon[29436]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-25T06:19:26.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:25 smithi175 ceph-mon[29436]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:25 smithi175 ceph-mon[24776]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-25T06:19:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:25 smithi175 ceph-mon[24776]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:26 smithi179 ceph-mon[25623]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:19:27.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:26 smithi175 ceph-mon[24776]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:19:27.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:26 smithi175 ceph-mon[29436]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:19:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:28 smithi179 ceph-mon[25623]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:29.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:28 smithi175 ceph-mon[24776]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:29.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:28 smithi175 ceph-mon[29436]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:31 smithi179 ceph-mon[25623]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:19:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:32.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:31 smithi175 ceph-mon[24776]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:19:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:31 smithi175 ceph-mon[29436]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-05-25T06:19:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:33 smithi179 ceph-mon[25623]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:19:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:19:33] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.33.4" 2024-05-25T06:19:34.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:33 smithi175 ceph-mon[24776]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:34.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:33 smithi175 ceph-mon[29436]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:19:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:35 smithi179 ceph-mon[25623]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:19:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:35 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:36.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:35 smithi175 ceph-mon[24776]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:19:36.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:35 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:35 smithi175 ceph-mon[29436]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:19:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:35 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:37 smithi179 ceph-mon[25623]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:38.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:37 smithi175 ceph-mon[24776]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:38.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:37 smithi175 ceph-mon[29436]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:39 smithi179 ceph-mon[25623]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:40.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:39 smithi175 ceph-mon[29436]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:39 smithi175 ceph-mon[24776]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:41 smithi179 ceph-mon[25623]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:42.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:41 smithi175 ceph-mon[29436]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:41 smithi175 ceph-mon[24776]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:43 smithi179 ceph-mon[25623]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:43.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:19:43 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:19:43] "GET /metrics HTTP/1.1" 200 38006 "" "Prometheus/2.33.4" 2024-05-25T06:19:44.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:43 smithi175 ceph-mon[24776]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:43 smithi175 ceph-mon[29436]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:45 smithi179 ceph-mon[25623]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:46.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:45 smithi175 ceph-mon[24776]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:45 smithi175 ceph-mon[29436]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:47 smithi179 ceph-mon[25623]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:48.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:47 smithi175 ceph-mon[29436]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:47 smithi175 ceph-mon[24776]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:49 smithi179 ceph-mon[25623]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:50.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:49 smithi175 ceph-mon[29436]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:50.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:49 smithi175 ceph-mon[24776]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:50 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:51.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:50 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:50 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:19:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:51 smithi179 ceph-mon[25623]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:51 smithi175 ceph-mon[29436]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:51 smithi175 ceph-mon[24776]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:19:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:53 smithi179 ceph-mon[25623]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:53.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:19:53 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:19:53] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.33.4" 2024-05-25T06:19:54.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:53 smithi175 ceph-mon[24776]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:53 smithi175 ceph-mon[29436]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:19:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:55 smithi179 ceph-mon[25623]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:56.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:55 smithi175 ceph-mon[24776]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:56.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:55 smithi175 ceph-mon[29436]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:57 smithi179 ceph-mon[25623]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:57 smithi175 ceph-mon[29436]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:19:58.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:57 smithi175 ceph-mon[24776]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:19:59 smithi179 ceph-mon[25623]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:19:59 smithi175 ceph-mon[29436]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:00.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:19:59 smithi175 ceph-mon[24776]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:00 smithi179 ceph-mon[25623]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:20:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:00 smithi179 ceph-mon[25623]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:20:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:00 smithi179 ceph-mon[25623]: daemon iscsi.foo.smithi175.buback on smithi175 is in error state 2024-05-25T06:20:01.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:00 smithi175 ceph-mon[24776]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:20:01.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:00 smithi175 ceph-mon[24776]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:20:01.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:00 smithi175 ceph-mon[24776]: daemon iscsi.foo.smithi175.buback on smithi175 is in error state 2024-05-25T06:20:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:00 smithi175 ceph-mon[29436]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:20:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:00 smithi175 ceph-mon[29436]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:20:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:00 smithi175 ceph-mon[29436]: daemon iscsi.foo.smithi175.buback on smithi175 is in error state 2024-05-25T06:20:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:01 smithi179 ceph-mon[25623]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:01 smithi175 ceph-mon[24776]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:01 smithi175 ceph-mon[29436]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:02.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:03 smithi179 ceph-mon[25623]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:03.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:20:03] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.33.4" 2024-05-25T06:20:04.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:03 smithi175 ceph-mon[24776]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:03 smithi175 ceph-mon[29436]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:05 smithi179 ceph-mon[25623]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:05 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:06.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:05 smithi175 ceph-mon[24776]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:06.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:05 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:05 smithi175 ceph-mon[29436]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:05 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:07 smithi179 ceph-mon[25623]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:07 smithi175 ceph-mon[24776]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:08.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:07 smithi175 ceph-mon[29436]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:08.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:20:09.857 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:09 smithi175 ceph-mon[24776]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:09.857 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:09 smithi175 ceph-mon[29436]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:09 smithi179 ceph-mon[25623]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:10.646 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:20:10.646 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (12m) 2m ago 20m 24.6M - 0.23.0 ba2b418f427c 1893f9da103a 2024-05-25T06:20:10.647 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (12m) 2m ago 19m 55.7M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:20:10.647 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 error 2m ago 19m 56.2M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:20:10.647 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283 running (14m) 2m ago 24m 493M - 19.0.0-2843-gea0684c0 f9a9809cafd6 2c455cbdf457 2024-05-25T06:20:10.647 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (9m) 2m ago 26m 72.0M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:20:10.647 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (26m) 2m ago 26m 69.7M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:20:10.647 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (25m) 2m ago 25m 53.2M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (25m) 2m ago 25m 51.1M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (12m) 2m ago 21m 18.3M - 1.3.1 1dbe0e931976 3fd7592e3fcc 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (12m) 2m ago 21m 18.9M - 1.3.1 1dbe0e931976 3142b4d09fa0 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (24m) 2m ago 24m 55.7M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (24m) 2m ago 24m 56.9M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (23m) 2m ago 23m 51.7M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (23m) 2m ago 23m 52.9M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (23m) 2m ago 23m 53.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:20:10.648 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (22m) 2m ago 22m 51.6M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:20:10.649 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (22m) 2m ago 22m 53.1M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:20:10.649 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (21m) 2m ago 21m 55.6M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:20:10.649 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (2m) 2m ago 21m 46.4M - 2.33.4 514e6a882f6e 35dc48baab92 2024-05-25T06:20:10.649 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (19m) 2m ago 19m 74.9M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:20:10.649 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (19m) 2m ago 19m 78.0M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:20:11.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ls' 2024-05-25T06:20:11.785 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:11 smithi175 ceph-mon[29436]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:11.785 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:11.785 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:11 smithi175 ceph-mon[29436]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:11.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:11 smithi175 ceph-mon[24776]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:11.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:11.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:11 smithi175 ceph-mon[24776]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:11 smithi179 ceph-mon[25623]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:11 smithi179 ceph-mon[25623]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi175=a;count:1 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:grafana ?:3000 1/1 2m ago 21m smithi179=a;count:1 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo ?:5000 0/1 3m ago 19m count:1 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:mgr 2/2 3m ago 24m smithi175=y;smithi179=x;count:2 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:mon 3/3 3m ago 25m smithi175:172.21.15.175=a;smithi175:[v2:172.21.15.175:3301,v1:172.21.15.175:6790]=c;smithi179:172.21.15.179=b;count:3 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi175=a;smithi179=b;count:2 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:osd 8 3m ago - 2024-05-25T06:20:13.285 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus ?:9095 1/1 2m ago 21m smithi179=a;count:1 2024-05-25T06:20:13.286 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-05-25T06:20:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:13 smithi179 ceph-mon[25623]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:13.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:20:13] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.33.4" 2024-05-25T06:20:13.938 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:13 smithi175 ceph-mon[24776]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:13.938 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:13 smithi175 ceph-mon[29436]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:13.962 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions' 2024-05-25T06:20:14.936 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:14 smithi175 ceph-mon[29436]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:14.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:14 smithi175 ceph-mon[24776]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:14 smithi179 ceph-mon[25623]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:15.976 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:20:15.976 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:15.977 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:20:15.978 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:20:15.979 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:15 smithi175 ceph-mon[24776]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:15.979 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:15 smithi175 ceph-mon[29436]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:15 smithi179 ceph-mon[25623]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:16.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-05-25T06:20:16.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:16 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/942944675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:16 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/942944675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:16 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/942944675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:17.977 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:17 smithi175 ceph-mon[29436]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:17.978 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:17 smithi175 ceph-mon[24776]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:17 smithi179 ceph-mon[25623]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:19.753 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:19 smithi175 ceph-mon[29436]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:19.753 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:19 smithi175 ceph-mon[29436]: from='client.25309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:19.754 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:19 smithi175 ceph-mon[24776]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:19.754 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:19 smithi175 ceph-mon[24776]: from='client.25309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:19 smithi179 ceph-mon[25623]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:20:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:19 smithi179 ceph-mon[25623]: from='client.25309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:20.655 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:20:21.011 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:21.011 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:21.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:20 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:20 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:20 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:20 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:20 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:20 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:20 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:21.278 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-25T06:20:21.909 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[29436]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:21.909 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:21.909 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[29436]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:20:21.909 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[29436]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-05-25T06:20:21.909 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[29436]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s 2024-05-25T06:20:21.910 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[29436]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:20:21.910 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[24776]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:21.910 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:21.910 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[24776]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:20:21.910 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[24776]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-05-25T06:20:21.910 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[24776]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s 2024-05-25T06:20:21.910 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:21 smithi175 ceph-mon[24776]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:20:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:21 smithi179 ceph-mon[25623]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:20:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:21 smithi179 ceph-mon[25623]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:20:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:21 smithi179 ceph-mon[25623]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-05-25T06:20:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:21 smithi179 ceph-mon[25623]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s 2024-05-25T06:20:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:21 smithi179 ceph-mon[25623]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:20:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:22 smithi179 ceph-mon[25623]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:20:23.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:22 smithi175 ceph-mon[24776]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:20:23.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:22 smithi175 ceph-mon[29436]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:20:23.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:20:23] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.33.4" 2024-05-25T06:20:23.805 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:20:24.466 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:20:24.466 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (12m) 3m ago 20m 24.6M - 0.23.0 ba2b418f427c 1893f9da103a 2024-05-25T06:20:24.466 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (12m) 3m ago 20m 55.7M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:20:24.466 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 error 3m ago 19m 56.2M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:20:24.466 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283 running (15m) 3m ago 25m 493M - 19.0.0-2843-gea0684c0 f9a9809cafd6 2c455cbdf457 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (9m) 3m ago 26m 72.0M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (26m) 3m ago 26m 69.7M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (25m) 3m ago 25m 53.2M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (25m) 3m ago 25m 51.1M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (12m) 3m ago 21m 18.3M - 1.3.1 1dbe0e931976 3fd7592e3fcc 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (12m) 3m ago 21m 18.9M - 1.3.1 1dbe0e931976 3142b4d09fa0 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (24m) 3m ago 24m 55.7M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (24m) 3m ago 24m 56.9M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (24m) 3m ago 23m 51.7M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (23m) 3m ago 23m 52.9M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (23m) 3m ago 23m 53.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (22m) 3m ago 22m 51.6M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (22m) 3m ago 22m 53.1M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:20:24.467 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (22m) 3m ago 22m 55.6M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:20:24.468 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (3m) 3m ago 21m 46.4M - 2.33.4 514e6a882f6e 35dc48baab92 2024-05-25T06:20:24.468 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (19m) 3m ago 19m 74.9M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:20:24.468 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (19m) 3m ago 19m 78.0M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:20:24.853 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:20:24.854 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:20:24.855 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:24.855 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:20:24.855 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:20:24.855 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:20:24.855 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:20:24.855 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:20:25.189 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:20:25.189 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:20:25.189 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:20:25.189 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-25T06:20:25.189 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [ 2024-05-25T06:20:25.189 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr" 2024-05-25T06:20:25.190 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:20:25.190 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "2/2 daemons upgraded", 2024-05-25T06:20:25.190 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 image", 2024-05-25T06:20:25.190 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:20:25.190 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:20:25.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:20:25] ENGINE Bus STOPPING 2024-05-25T06:20:25.444 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:20:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:20:25] ENGINE Bus STOPPED 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:20:25] ENGINE Bus STARTING 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='client.25315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='client.15441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:25.445 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: Failing over to other MGR 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='client.25327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 0 op/s 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2613147910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='client.25315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.446 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='client.15441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: Failing over to other MGR 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='client.25327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 0 op/s 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2613147910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:25.447 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:25 smithi175 ceph-mon[24776]: Standby manager daemon x started 2024-05-25T06:20:25.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:25.252+0000 7fdcfd5c6700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-25T06:20:25.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ignoring --setuser ceph since I am not root 2024-05-25T06:20:25.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ignoring --setgroup ceph since I am not root 2024-05-25T06:20:25.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mgr[45727]: -- 172.21.15.179:0/1121096486 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5555c3cd05a0 con 0x5555c3ca5000 2024-05-25T06:20:25.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:25.433+0000 7fb52c0e1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='client.25315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='client.15441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: Failing over to other MGR 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='mgr.25022 172.21.15.179:0/975186863' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='client.25327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:25.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 0 op/s 2024-05-25T06:20:25.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2613147910' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:25.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:25 smithi179 ceph-mon[25623]: Standby manager daemon x started 2024-05-25T06:20:25.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:20:25] ENGINE Serving on http://:::9283 2024-05-25T06:20:25.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:20:25] ENGINE Bus STARTED 2024-05-25T06:20:26.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:25.625+0000 7fb52c0e1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:20:26.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:25.737+0000 7fb52c0e1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:20:26.413 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:26.067+0000 7fb52c0e1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:20:26.413 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:26.173+0000 7fb52c0e1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:20:26.413 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.25022 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-25T06:20:26.413 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: mgrmap e35: y(active, starting, since 1.00205s), standbys: x 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:20:26.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: Manager daemon y is now available 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:20:26.415 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:26 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:20:26.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.25022 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-25T06:20:26.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: mgrmap e35: y(active, starting, since 1.00205s), standbys: x 2024-05-25T06:20:26.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:20:26.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:20:26.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: Manager daemon y is now available 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.25022 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: mgrmap e35: y(active, starting, since 1.00205s), standbys: x 2024-05-25T06:20:26.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:20:26.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: Manager daemon y is now available 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:20:26.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:26 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:20:27.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[24776]: [25/May/2024:06:20:25] ENGINE Bus STARTING 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[24776]: [25/May/2024:06:20:26] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[24776]: [25/May/2024:06:20:26] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[24776]: [25/May/2024:06:20:26] ENGINE Bus STARTED 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[24776]: mgrmap e36: y(active, since 2s), standbys: x 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[24776]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[29436]: [25/May/2024:06:20:25] ENGINE Bus STARTING 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[29436]: [25/May/2024:06:20:26] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[29436]: [25/May/2024:06:20:26] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[29436]: [25/May/2024:06:20:26] ENGINE Bus STARTED 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[29436]: mgrmap e36: y(active, since 2s), standbys: x 2024-05-25T06:20:27.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:27 smithi175 ceph-mon[29436]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:27.425 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:27 smithi179 ceph-mon[25623]: [25/May/2024:06:20:25] ENGINE Bus STARTING 2024-05-25T06:20:27.425 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:27 smithi179 ceph-mon[25623]: [25/May/2024:06:20:26] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:20:27.425 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:27 smithi179 ceph-mon[25623]: [25/May/2024:06:20:26] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:20:27.425 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:27 smithi179 ceph-mon[25623]: [25/May/2024:06:20:26] ENGINE Bus STARTED 2024-05-25T06:20:27.425 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:27 smithi179 ceph-mon[25623]: mgrmap e36: y(active, since 2s), standbys: x 2024-05-25T06:20:27.425 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:27 smithi179 ceph-mon[25623]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:27.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:27.424+0000 7fb52c0e1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:20:27.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:27.620+0000 7fb52c0e1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:20:28.116 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:27.834+0000 7fb52c0e1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:20:28.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:28 smithi175 ceph-mon[24776]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:20:28.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:28 smithi175 ceph-mon[24776]: Cluster is now healthy 2024-05-25T06:20:28.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:28 smithi175 ceph-mon[24776]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:28.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:28 smithi175 ceph-mon[29436]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:20:28.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:28 smithi175 ceph-mon[29436]: Cluster is now healthy 2024-05-25T06:20:28.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:28 smithi175 ceph-mon[29436]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:28.466 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:28 smithi179 ceph-mon[25623]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:20:28.466 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:28 smithi179 ceph-mon[25623]: Cluster is now healthy 2024-05-25T06:20:28.466 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:28 smithi179 ceph-mon[25623]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:28.466 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:28.142+0000 7fb52c0e1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:20:28.467 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:28.236+0000 7fb52c0e1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:20:28.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:28.465+0000 7fb52c0e1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:20:28.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:28.675+0000 7fb52c0e1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:20:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:29 smithi179 ceph-mon[25623]: mgrmap e37: y(active, since 4s), standbys: x 2024-05-25T06:20:29.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:29.310+0000 7fb52c0e1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:20:29.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:29 smithi175 ceph-mon[29436]: mgrmap e37: y(active, since 4s), standbys: x 2024-05-25T06:20:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:29 smithi175 ceph-mon[24776]: mgrmap e37: y(active, since 4s), standbys: x 2024-05-25T06:20:29.870 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:29.761+0000 7fb52c0e1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:20:29.871 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:29.870+0000 7fb52c0e1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:20:30.286 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:30.089+0000 7fb52c0e1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:20:31.015 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:30 smithi179 ceph-mon[25623]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:31.015 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:30 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.015 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:30 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.015 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:30 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.015 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:30 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.015 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:30.649+0000 7fb52c0e1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:20:31.015 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:30.750+0000 7fb52c0e1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:20:31.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[24776]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:31.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[29436]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:30 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:31.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:31.014+0000 7fb52c0e1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:20:31.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:31.160+0000 7fb52c0e1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:20:31.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:31.264+0000 7fb52c0e1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:20:31.712 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:31.391+0000 7fb52c0e1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:31.712+0000 7fb52c0e1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:31 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:31 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:31 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:31 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:31 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:31 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:32.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:32.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:31 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:32.670 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:32.340+0000 7fb52c0e1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:20:32.671 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:32.563+0000 7fb52c0e1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:20:32.671 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:20:32] ENGINE Bus STARTING 2024-05-25T06:20:32.671 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: CherryPy Checker: 2024-05-25T06:20:32.671 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: The Application mounted at '' has an empty config. 2024-05-25T06:20:32.671 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: 2024-05-25T06:20:32.671 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:20:32] ENGINE Serving on http://:::9283 2024-05-25T06:20:33.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: [25/May/2024:06:20:32] ENGINE Bus STARTED 2024-05-25T06:20:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:20:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:20:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:20:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:33.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:33.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Standby manager daemon x restarted 2024-05-25T06:20:33.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: Standby manager daemon x started 2024-05-25T06:20:33.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:20:33.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:20:33.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:20:33.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:32 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:20:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:20:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:20:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:20:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:33.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Standby manager daemon x restarted 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:20:33.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Standby manager daemon x restarted 2024-05-25T06:20:33.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: Standby manager daemon x started 2024-05-25T06:20:33.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:20:33.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:20:33.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:20:33.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:32 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/3731731619' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:20:33.779 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[45723]: ::ffff:172.21.15.179 - - [25/May/2024:06:20:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-25T06:20:34.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:33 smithi179 ceph-mon[25623]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-25T06:20:34.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:33 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:20:34.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:33 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:20:34.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:33 smithi179 ceph-mon[25623]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:20:34.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:33 smithi179 ceph-mon[25623]: mgrmap e38: y(active, since 8s), standbys: x 2024-05-25T06:20:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[24776]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-25T06:20:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:20:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:20:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[24776]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:20:34.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[24776]: mgrmap e38: y(active, since 8s), standbys: x 2024-05-25T06:20:34.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[29436]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-25T06:20:34.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:20:34.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:20:34.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[29436]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:20:34.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:33 smithi175 ceph-mon[29436]: mgrmap e38: y(active, since 8s), standbys: x 2024-05-25T06:20:34.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:20:33.882+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:34.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:34.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:34.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:20:34.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:34.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:34.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:34.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:34.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:34.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:35 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:36.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:36.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:36.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:36.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:35 smithi179 ceph-mon[25623]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:20:36.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:35 smithi179 ceph-mon[25623]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-25T06:20:36.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:35 smithi179 ceph-mon[25623]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:20:36.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:36.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:36.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:36.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:36.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:36.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:36.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:36.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:36.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:36.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:36.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:20:36.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:36.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:36.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:36.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:36.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:36.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:36.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:36.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:36.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:36.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:36.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:36.080 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:36.080 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[24776]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[24776]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[24776]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:36.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:36.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[29436]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-25T06:20:36.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[29436]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-25T06:20:36.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:35 smithi175 ceph-mon[29436]: Reconfiguring daemon prometheus.a on smithi179 2024-05-25T06:20:37.482 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 systemd[1]: Stopping Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:20:37.483 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 bash[59972]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:20:37.783 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.482Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:20:37.783 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-25T06:20:37.783 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-25T06:20:37.783 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[57193]: ts=2024-05-25T06:20:37.483Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-25T06:20:37.784 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 podman[59983]: 2024-05-25 06:20:37.493312245 +0000 UTC m=+0.107994566 container died 35dc48baab92354e0038228ba53872195b343be75a3e9530db8c79b47ad5def0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:37 smithi179 ceph-mon[25623]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:20:38.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 podman[59983]: 2024-05-25 06:20:37.866277358 +0000 UTC m=+0.480959675 container cleanup 35dc48baab92354e0038228ba53872195b343be75a3e9530db8c79b47ad5def0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:38.036 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 bash[59983]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a 2024-05-25T06:20:38.036 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:37 smithi179 bash[60009]: Error: no container with name or ID "ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus.a" found: no such container 2024-05-25T06:20:38.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:37 smithi175 ceph-mon[24776]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:20:38.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:37 smithi175 ceph-mon[29436]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:20:38.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 podman[59997]: 2024-05-25 06:20:38.134244372 +0000 UTC m=+0.638248650 container remove 35dc48baab92354e0038228ba53872195b343be75a3e9530db8c79b47ad5def0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:38.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-25T06:20:38.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 systemd[1]: Stopped Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:20:38.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Consumed 1.421s CPU time. 2024-05-25T06:20:38.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 systemd[1]: Starting Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:20:38.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 podman[60071]: 2024-05-25 06:20:38.264169401 +0000 UTC m=+0.018094705 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-25T06:20:38.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 podman[60071]: 2024-05-25 06:20:38.368353421 +0000 UTC m=+0.122278713 container create 254b9d2cd5d342a15cc65ba28d184cbc63383e94560b3bb9ed53d4e6a01c4546 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 podman[60071]: 2024-05-25 06:20:38.67921861 +0000 UTC m=+0.433143916 container init 254b9d2cd5d342a15cc65ba28d184cbc63383e94560b3bb9ed53d4e6a01c4546 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 podman[60071]: 2024-05-25 06:20:38.682170319 +0000 UTC m=+0.436095613 container start 254b9d2cd5d342a15cc65ba28d184cbc63383e94560b3bb9ed53d4e6a01c4546 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.706Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.706Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.706Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.706Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi179 (none))" 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.706Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-25T06:20:38.922 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.706Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.708Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.708Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.709Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.710Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.711Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=735.401µs 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.711Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 bash[60071]: 254b9d2cd5d342a15cc65ba28d184cbc63383e94560b3bb9ed53d4e6a01c4546 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 systemd[1]: Started Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:20:38.923 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:38 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:38.922Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-05-25T06:20:39.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.143Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-05-25T06:20:39.588 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.338Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.397Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.404Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.405Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.405Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=45.03µs wal_replay_duration=693.500335ms total_replay_duration=694.299417ms 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.411Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.411Z caller=main.go:947 level=info msg="TSDB started" 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.411Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.430Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.940298ms db_storage=710ns remote_storage=1.241µs web_handler=728ns query_engine=699ns scrape=384.959µs scrape_sd=93.874µs notify=10.984µs notify_sd=9.087µs rules=18.125222ms 2024-05-25T06:20:39.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:20:39 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:20:39.430Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-25T06:20:39.992 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:39 smithi179 ceph-mon[25623]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 77 op/s 2024-05-25T06:20:39.992 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:39 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:39.992 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:39 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:39.993 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:39 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:20:39.993 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:39 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[24776]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 77 op/s 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[29436]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 77 op/s 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:40.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:20:40.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:39 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:40 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:20:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:40 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:40 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:20:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:40 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:20:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:40 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:20:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:40 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:41.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:20:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:40 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:41.911 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:41 smithi179 ceph-mon[25623]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 77 op/s 2024-05-25T06:20:41.911 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:41.912 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:41 smithi179 ceph-mon[25623]: Upgrade: Updating mgr.x 2024-05-25T06:20:41.912 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:41 smithi179 ceph-mon[25623]: Deploying daemon mgr.x on smithi179 2024-05-25T06:20:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[24776]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 77 op/s 2024-05-25T06:20:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[24776]: Upgrade: Updating mgr.x 2024-05-25T06:20:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[24776]: Deploying daemon mgr.x on smithi179 2024-05-25T06:20:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[29436]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 77 op/s 2024-05-25T06:20:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[29436]: Upgrade: Updating mgr.x 2024-05-25T06:20:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:41 smithi175 ceph-mon[29436]: Deploying daemon mgr.x on smithi179 2024-05-25T06:20:42.444 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:42 smithi179 systemd[1]: Stopping Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:20:42.714 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:42 smithi179 podman[60425]: 2024-05-25 06:20:42.444795809 +0000 UTC m=+0.113907762 container died 2c455cbdf4573339ce2f9bb9ffc37fffc5426cb4f4aca988d559dc94758b352f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux ) 2024-05-25T06:20:43.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:42 smithi179 podman[60425]: 2024-05-25 06:20:42.788998168 +0000 UTC m=+0.458110162 container cleanup 2c455cbdf4573339ce2f9bb9ffc37fffc5426cb4f4aca988d559dc94758b352f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-25T06:20:43.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:42 smithi179 bash[60425]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x 2024-05-25T06:20:43.380 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 podman[60440]: 2024-05-25 06:20:43.065096198 +0000 UTC m=+0.619044492 container remove 2c455cbdf4573339ce2f9bb9ffc37fffc5426cb4f4aca988d559dc94758b352f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:20:43.380 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:20:43.380 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-25T06:20:43.380 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 systemd[1]: Stopped Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:20:43.380 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Consumed 59.014s CPU time. 2024-05-25T06:20:43.380 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 systemd[1]: Starting Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:20:43.632 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 podman[60566]: 2024-05-25 06:20:43.497429857 +0000 UTC m=+0.016944972 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:20:43.893 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:43 smithi179 ceph-mon[25623]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 91 op/s 2024-05-25T06:20:43.893 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 podman[60566]: 2024-05-25 06:20:43.63319309 +0000 UTC m=+0.152708203 container create ab279f96819df27da2850ac855d4da176e252e9723ce0c784a19b5a4bbcdcf2f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-25T06:20:44.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:43 smithi175 ceph-mon[24776]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 91 op/s 2024-05-25T06:20:44.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:43 smithi175 ceph-mon[29436]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 91 op/s 2024-05-25T06:20:44.254 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 podman[60566]: 2024-05-25 06:20:43.894002476 +0000 UTC m=+0.413517580 container init ab279f96819df27da2850ac855d4da176e252e9723ce0c784a19b5a4bbcdcf2f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-05-25T06:20:44.254 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 podman[60566]: 2024-05-25 06:20:43.897149119 +0000 UTC m=+0.416664211 container start ab279f96819df27da2850ac855d4da176e252e9723ce0c784a19b5a4bbcdcf2f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True) 2024-05-25T06:20:44.254 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 ceph-mgr[60585]: -- 172.21.15.179:0/3672977136 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b80f2405a0 con 0x55b80f215000 2024-05-25T06:20:44.254 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 bash[60566]: ab279f96819df27da2850ac855d4da176e252e9723ce0c784a19b5a4bbcdcf2f 2024-05-25T06:20:44.254 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:43 smithi179 systemd[1]: Started Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:20:44.254 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:44 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:44.058+0000 7f5655ee7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:20:44.254 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:44 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:44.253+0000 7f5655ee7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:20:44.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:44 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:44.361+0000 7f5655ee7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:20:45.010 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:44 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:44.711+0000 7f5655ee7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:20:45.010 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:44 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:44.801+0000 7f5655ee7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:20:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:45 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:45 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:45 smithi179 ceph-mon[25623]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-25T06:20:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:45 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[24776]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[29436]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-25T06:20:45.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:45 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:46.401 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:46.067+0000 7f5655ee7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:20:46.402 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:46.269+0000 7f5655ee7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:20:46.755 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:46.453+0000 7f5655ee7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:20:47.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:46.754+0000 7f5655ee7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:20:47.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:46 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:46.852+0000 7f5655ee7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:20:47.356 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:47 smithi179 ceph-mon[25623]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-25T06:20:47.356 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:47.068+0000 7f5655ee7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:20:47.357 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:47.272+0000 7f5655ee7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:20:47.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:47 smithi175 ceph-mon[29436]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-25T06:20:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:47 smithi175 ceph-mon[24776]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-25T06:20:48.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:47.918+0000 7f5655ee7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:20:48.687 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:48.359+0000 7f5655ee7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:20:48.687 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:48.470+0000 7f5655ee7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:20:48.687 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:48.686+0000 7f5655ee7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:20:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:49 smithi179 ceph-mon[25623]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:20:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:49 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:49 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:49.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:49.247+0000 7f5655ee7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:20:49.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:49.346+0000 7f5655ee7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:20:49.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:49 smithi175 ceph-mon[29436]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:20:49.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:49 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:49.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:49 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:49 smithi175 ceph-mon[24776]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-25T06:20:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:49 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:49.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:49 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:49.990 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:49.620+0000 7f5655ee7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:20:49.990 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:49.764+0000 7f5655ee7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:20:49.991 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:49.865+0000 7f5655ee7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:20:49.991 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:49.991+0000 7f5655ee7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:20:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:50 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:50 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:50.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:50 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:50.315+0000 7f5655ee7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:20:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:50 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:50.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:50 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:50.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:50 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:50.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:50 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:51.202 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:50 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:50.946+0000 7f5655ee7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:20:51.478 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:51.202+0000 7f5655ee7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:20:51.478 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: [25/May/2024:06:20:51] ENGINE Bus STARTING 2024-05-25T06:20:51.479 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: CherryPy Checker: 2024-05-25T06:20:51.479 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: The Application mounted at '' has an empty config. 2024-05-25T06:20:51.479 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:20:51.479 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: [25/May/2024:06:20:51] ENGINE Serving on http://:::9283 2024-05-25T06:20:51.479 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:20:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: [25/May/2024:06:20:51] ENGINE Bus STARTED 2024-05-25T06:20:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:20:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: Standby manager daemon x restarted 2024-05-25T06:20:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: Standby manager daemon x started 2024-05-25T06:20:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:20:51.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:20:51.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:20:51.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:51 smithi179 ceph-mon[25623]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: Standby manager daemon x restarted 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:20:52.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:20:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:20:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: Standby manager daemon x restarted 2024-05-25T06:20:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: Standby manager daemon x started 2024-05-25T06:20:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:20:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:20:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:20:52.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:51 smithi175 ceph-mon[24776]: from='mgr.? 172.21.15.179:0/4218391776' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:20:52.925 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:52 smithi179 ceph-mon[25623]: mgrmap e39: y(active, since 27s), standbys: x 2024-05-25T06:20:52.990 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:52 smithi175 ceph-mon[29436]: mgrmap e39: y(active, since 27s), standbys: x 2024-05-25T06:20:52.990 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:52 smithi175 ceph-mon[24776]: mgrmap e39: y(active, since 27s), standbys: x 2024-05-25T06:20:53.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:20:53] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.33.4" 2024-05-25T06:20:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:20:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:53.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:53 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:20:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:20:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:54.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:20:54.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:53 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:20:54.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[29436]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 52 op/s 2024-05-25T06:20:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[29436]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-05-25T06:20:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:20:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:20:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[24776]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 52 op/s 2024-05-25T06:20:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[24776]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-05-25T06:20:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:20:54.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:54 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:20:54.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:20:54.680+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:54.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:20:54.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:54.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:54.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:54.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:54.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:54.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:54.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:20:54.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:54.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:20:54.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:54.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:54.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:54.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:54.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:54.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:54.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:54.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:54.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:20:54 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:54 smithi179 ceph-mon[25623]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 52 op/s 2024-05-25T06:20:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:54 smithi179 ceph-mon[25623]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-05-25T06:20:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:54 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:20:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:54 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:20:55.566 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:20:56.007 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:56.007 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:56.008 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:56.009 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:56.010 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:56.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:56.012 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:56.013 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:56.014 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:56.015 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all mgr 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-25T06:20:56.016 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all crash 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all mds 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.017 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all mgr 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.018 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all crash 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all mds 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:20:56.019 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all nfs 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.020 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all nfs 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:56.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:20:56.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:20:56.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:20:56.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all mgr 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all crash 2024-05-25T06:20:56.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all mds 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all nfs 2024-05-25T06:20:56.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:20:56.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:20:56.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:20:56.237 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (13m) 26s ago 21m 24.7M - 0.23.0 ba2b418f427c 1893f9da103a 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (13m) 7s ago 20m 55.9M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 error 26s ago 20m 58.9M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (12s) 7s ago 25m 65.2M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (10m) 26s ago 27m 478M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (27m) 26s ago 27m 73.5M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (25m) 7s ago 25m 54.9M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (26m) 26s ago 26m 51.7M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (13m) 26s ago 21m 18.2M - 1.3.1 1dbe0e931976 3fd7592e3fcc 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (13m) 7s ago 21m 21.1M - 1.3.1 1dbe0e931976 3142b4d09fa0 2024-05-25T06:20:56.238 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (25m) 26s ago 25m 56.2M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (24m) 26s ago 24m 57.2M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (24m) 26s ago 24m 51.9M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (24m) 26s ago 24m 52.9M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (23m) 7s ago 23m 55.6M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (23m) 7s ago 23m 52.4M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (22m) 7s ago 22m 54.6M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (22m) 7s ago 22m 56.7M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (17s) 7s ago 21m 53.4M - 2.33.4 514e6a882f6e 254b9d2cd5d3 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (20m) 26s ago 20m 75.8M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:20:56.239 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (20m) 7s ago 20m 78.8M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:20:56.647 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:20:56.647 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:20:56.647 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:20:56.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:20:56.648 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:20:56.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:56 smithi175 ceph-mon[29436]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-05-25T06:20:56.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:56 smithi175 ceph-mon[29436]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:56.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:56 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1381008414' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:56.941 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:56 smithi175 ceph-mon[24776]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-05-25T06:20:56.941 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:56 smithi175 ceph-mon[24776]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:56.941 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:56 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1381008414' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [ 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr" 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "2/2 daemons upgraded", 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading node-exporter daemons", 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:20:57.004 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:20:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:56 smithi179 ceph-mon[25623]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-05-25T06:20:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:56 smithi179 ceph-mon[25623]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:56 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1381008414' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:20:57.319 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:57 smithi175 systemd[1]: Stopping Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:20:57.705 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:57 smithi175 podman[75890]: 2024-05-25 06:20:57.416024787 +0000 UTC m=+0.139350037 container died 3fd7592e3fcc03dee13fd956ce62d457fab9c29222ffc8ebd88d44cba7bce868 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:57 smithi179 ceph-mon[25623]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:57 smithi179 ceph-mon[25623]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:57 smithi179 ceph-mon[25623]: Upgrade: Updating node-exporter.a (1/2) 2024-05-25T06:20:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:57 smithi179 ceph-mon[25623]: Deploying daemon node-exporter.a on smithi175 2024-05-25T06:20:58.038 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:57 smithi175 podman[75890]: 2024-05-25 06:20:57.805006659 +0000 UTC m=+0.528331898 container cleanup 3fd7592e3fcc03dee13fd956ce62d457fab9c29222ffc8ebd88d44cba7bce868 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:58.038 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:57 smithi175 bash[75890]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[29436]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[29436]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[29436]: Upgrade: Updating node-exporter.a (1/2) 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[29436]: Deploying daemon node-exporter.a on smithi175 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[24776]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[24776]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[24776]: Upgrade: Updating node-exporter.a (1/2) 2024-05-25T06:20:58.039 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:57 smithi175 ceph-mon[24776]: Deploying daemon node-exporter.a on smithi175 2024-05-25T06:20:58.319 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 podman[75905]: 2024-05-25 06:20:58.03955237 +0000 UTC m=+0.621505314 container remove 3fd7592e3fcc03dee13fd956ce62d457fab9c29222ffc8ebd88d44cba7bce868 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:20:58.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:20:58.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-25T06:20:58.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 systemd[1]: Stopped Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:20:58.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Consumed 5.223s CPU time. 2024-05-25T06:20:58.775 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 systemd[1]: Starting Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:20:58.775 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 bash[76032]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-25T06:20:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:58 smithi179 ceph-mon[25623]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:20:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:20:58 smithi179 ceph-mon[25623]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:59.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:58 smithi175 ceph-mon[29436]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:20:59.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:20:58 smithi175 ceph-mon[29436]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:58 smithi175 ceph-mon[24776]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:20:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:20:58 smithi175 ceph-mon[24776]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:20:59.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 bash[76032]: Getting image source signatures 2024-05-25T06:20:59.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 bash[76032]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-25T06:20:59.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 bash[76032]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-25T06:20:59.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:20:58 smithi175 bash[76032]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-25T06:21:00.569 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 bash[76032]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-25T06:21:00.570 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 bash[76032]: Writing manifest to image destination 2024-05-25T06:21:00.962 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:00 smithi175 ceph-mon[29436]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:00.962 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-mon[24776]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:00.962 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 podman[76032]: 2024-05-25 06:21:00.654173053 +0000 UTC m=+2.165629130 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-25T06:21:00.963 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 podman[76032]: 2024-05-25 06:21:00.752605884 +0000 UTC m=+2.264061975 container create e713992d438246b355ad47fb662cd36d5e78dbb72ed8effa57d25cabf29f5d42 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:00.963 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 podman[76032]: 2024-05-25 06:21:00.962990261 +0000 UTC m=+2.474446335 container init e713992d438246b355ad47fb662cd36d5e78dbb72ed8effa57d25cabf29f5d42 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:00 smithi179 ceph-mon[25623]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 podman[76032]: 2024-05-25 06:21:00.965853461 +0000 UTC m=+2.477309531 container start e713992d438246b355ad47fb662cd36d5e78dbb72ed8effa57d25cabf29f5d42 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.969Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.969Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-25T06:21:01.320 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=node_exporter.go:117 level=info collector=arp 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.970Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-25T06:21:01.321 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=edac 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-25T06:21:01.322 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=os 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-25T06:21:01.323 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=stat 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=time 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=uname 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a[76084]: ts=2024-05-25T06:21:00.971Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:01 smithi175 bash[76032]: e713992d438246b355ad47fb662cd36d5e78dbb72ed8effa57d25cabf29f5d42 2024-05-25T06:21:01.324 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:21:01 smithi175 systemd[1]: Started Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:21:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:01 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:01 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:01 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:01 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:01 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:01 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:02 smithi179 ceph-mon[25623]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 575 B/s rd, 0 op/s 2024-05-25T06:21:03.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:02 smithi175 ceph-mon[24776]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 575 B/s rd, 0 op/s 2024-05-25T06:21:03.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:02 smithi175 ceph-mon[29436]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 575 B/s rd, 0 op/s 2024-05-25T06:21:03.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:03 smithi179 systemd[1]: Stopping Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:21:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:03 smithi175 ceph-mon[24776]: Upgrade: Updating node-exporter.b (2/2) 2024-05-25T06:21:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:03 smithi175 ceph-mon[24776]: Deploying daemon node-exporter.b on smithi179 2024-05-25T06:21:03.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:21:03] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-05-25T06:21:03.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:03 smithi175 ceph-mon[29436]: Upgrade: Updating node-exporter.b (2/2) 2024-05-25T06:21:03.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:03 smithi175 ceph-mon[29436]: Deploying daemon node-exporter.b on smithi179 2024-05-25T06:21:03.912 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:03 smithi179 ceph-mon[25623]: Upgrade: Updating node-exporter.b (2/2) 2024-05-25T06:21:03.912 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:03 smithi179 ceph-mon[25623]: Deploying daemon node-exporter.b on smithi179 2024-05-25T06:21:03.912 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:03 smithi179 podman[61944]: 2024-05-25 06:21:03.647360383 +0000 UTC m=+0.111135059 container died 3142b4d09fa063ee011d014aa7826ac67184d79bce2dfc61df3f3f205ad5ddd5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:21:04.254 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 podman[61944]: 2024-05-25 06:21:04.003584601 +0000 UTC m=+0.467359276 container cleanup 3142b4d09fa063ee011d014aa7826ac67184d79bce2dfc61df3f3f205ad5ddd5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:21:04.254 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 bash[61944]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b 2024-05-25T06:21:04.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 podman[61955]: 2024-05-25 06:21:04.254672739 +0000 UTC m=+0.610015538 container remove 3142b4d09fa063ee011d014aa7826ac67184d79bce2dfc61df3f3f205ad5ddd5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:21:04.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:21:04.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-25T06:21:04.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 systemd[1]: Stopped Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:21:04.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Consumed 5.281s CPU time. 2024-05-25T06:21:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:04 smithi179 ceph-mon[25623]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:05.035 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 systemd[1]: Starting Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:21:05.035 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:04 smithi179 bash[62084]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-25T06:21:05.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:04 smithi175 ceph-mon[24776]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:05.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:04 smithi175 ceph-mon[29436]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:05.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:05 smithi179 bash[62084]: Getting image source signatures 2024-05-25T06:21:05.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:05 smithi179 bash[62084]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-25T06:21:05.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:05 smithi179 bash[62084]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-25T06:21:05.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:05 smithi179 bash[62084]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-25T06:21:07.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:06 smithi175 ceph-mon[24776]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:07.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:06 smithi175 ceph-mon[29436]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:07.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:06 smithi179 ceph-mon[25623]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:07.285 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:06 smithi179 bash[62084]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-25T06:21:07.285 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 bash[62084]: Writing manifest to image destination 2024-05-25T06:21:07.709 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 podman[62084]: 2024-05-25 06:21:07.292931702 +0000 UTC m=+2.560152678 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 podman[62084]: 2024-05-25 06:21:07.383046475 +0000 UTC m=+2.650267452 container create a3e7fafe20dff24a37ef15276ba93d6b82e08a7418670db640ffb46968833d3e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 podman[62084]: 2024-05-25 06:21:07.593589502 +0000 UTC m=+2.860810491 container init a3e7fafe20dff24a37ef15276ba93d6b82e08a7418670db640ffb46968833d3e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 podman[62084]: 2024-05-25 06:21:07.596588956 +0000 UTC m=+2.863809951 container start a3e7fafe20dff24a37ef15276ba93d6b82e08a7418670db640ffb46968833d3e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.600Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.600Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.601Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.601Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.601Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.601Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-25T06:21:07.710 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=arp 2024-05-25T06:21:07.711 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-25T06:21:07.711 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-25T06:21:07.711 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-25T06:21:07.711 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-25T06:21:07.711 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-25T06:21:07.711 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-25T06:21:07.711 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=edac 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-25T06:21:07.712 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=os 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-25T06:21:07.713 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=stat 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=time 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=uname 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b[62135]: ts=2024-05-25T06:21:07.602Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-25T06:21:07.714 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 bash[62084]: a3e7fafe20dff24a37ef15276ba93d6b82e08a7418670db640ffb46968833d3e 2024-05-25T06:21:08.035 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:21:07 smithi179 systemd[1]: Started Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:21:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:08 smithi179 ceph-mon[25623]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:08 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:08 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:08 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:09.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[24776]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:09.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:09.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:09.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[29436]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:08 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:10 smithi175 ceph-mon[24776]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:10 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:10 smithi175 ceph-mon[29436]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:10 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:11.022 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:10 smithi179 ceph-mon[25623]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:11.022 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:10 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:12 smithi179 ceph-mon[25623]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:12 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:12 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:12 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:12 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[29436]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[24776]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:12 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:13.810 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:21:13] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-05-25T06:21:14.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[24776]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[29436]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:14 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:14 smithi179 ceph-mon[25623]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:14 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:14 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:14 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:14 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:21:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:16 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:17.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:17.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:17.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:16 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[24776]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[24776]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[29436]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[29436]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:21:17.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:17 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:21:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:17 smithi179 ceph-mon[25623]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:21:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:17 smithi179 ceph-mon[25623]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:17 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:21:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:17 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:21:18.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:21:18.041+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:18.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:21:18.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:18.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:18.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:18.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:18.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:18.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:18.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:21:18.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:18.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:18.183 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:18.184 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:18.184 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:18.917 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:18 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:18 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:19 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:20.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:20.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:21:20.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:21:20.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:19 smithi179 ceph-mon[25623]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:20.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:19 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:20.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:20.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:20.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:20.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:19 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:20.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:21:20.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:20.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:20.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:19 smithi175 ceph-mon[24776]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:20.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:20.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:19 smithi175 ceph-mon[29436]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:20 smithi179 ceph-mon[25623]: Upgrade: Updating prometheus.a 2024-05-25T06:21:21.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:20 smithi175 ceph-mon[24776]: Upgrade: Updating prometheus.a 2024-05-25T06:21:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:20 smithi175 ceph-mon[29436]: Upgrade: Updating prometheus.a 2024-05-25T06:21:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:21 smithi175 ceph-mon[24776]: Deploying daemon prometheus.a on smithi179 2024-05-25T06:21:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:21 smithi175 ceph-mon[24776]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:21 smithi175 ceph-mon[29436]: Deploying daemon prometheus.a on smithi179 2024-05-25T06:21:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:21 smithi175 ceph-mon[29436]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:21 smithi179 ceph-mon[25623]: Deploying daemon prometheus.a on smithi179 2024-05-25T06:21:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:21 smithi179 ceph-mon[25623]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:23 smithi175 ceph-mon[24776]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:23.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:21:23] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-05-25T06:21:23.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:23 smithi175 ceph-mon[29436]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:23 smithi179 ceph-mon[25623]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:25 smithi179 ceph-mon[25623]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:25 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:25 smithi175 ceph-mon[24776]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:25 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:26.073 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:25 smithi175 ceph-mon[29436]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:21:26.073 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:25 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:27.363 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:21:27.988 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:27 smithi179 ceph-mon[25623]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:28.006 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:27 smithi175 ceph-mon[29436]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:28.006 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:21:28.006 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (13m) 15s ago 21m 24.8M - 0.23.0 ba2b418f427c 1893f9da103a 2024-05-25T06:21:28.006 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (13m) 16s ago 21m 55.6M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:21:28.006 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 15s ago 20m 59.1M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:21:28.006 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (44s) 16s ago 26m 450M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:21:28.006 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (10m) 15s ago 27m 503M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (27m) 15s ago 27m 77.8M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (26m) 16s ago 26m 55.7M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (26m) 15s ago 26m 56.3M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (27s) 15s ago 22m 7960k - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (20s) 16s ago 22m 3343k - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (25m) 15s ago 25m 56.9M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (25m) 15s ago 25m 58.4M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (25m) 15s ago 25m 52.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (24m) 15s ago 24m 53.0M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (24m) 16s ago 24m 55.7M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (23m) 16s ago 23m 52.4M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (23m) 16s ago 23m 54.6M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (23m) 16s ago 23m 56.8M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:21:28.007 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (49s) 16s ago 22m 64.7M - 2.33.4 514e6a882f6e 254b9d2cd5d3 2024-05-25T06:21:28.008 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (20m) 15s ago 20m 76.2M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:21:28.008 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (20m) 16s ago 20m 78.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:21:28.008 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:27 smithi175 ceph-mon[24776]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:28.404 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:21:28.404 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:21:28.405 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:21:28.406 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:21:28.406 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:21:28.406 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:21:28.406 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:21:28.406 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:21:28.406 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:21:28.406 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [ 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr" 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "2/2 daemons upgraded", 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading prometheus daemons", 2024-05-25T06:21:28.737 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:21:28.738 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:21:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:28 smithi179 ceph-mon[25623]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:28 smithi179 ceph-mon[25623]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:28 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/443999676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:29.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:28 smithi175 ceph-mon[24776]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:29.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:28 smithi175 ceph-mon[24776]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:29.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:28 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/443999676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:29.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:28 smithi175 ceph-mon[29436]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:29.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:28 smithi175 ceph-mon[29436]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:29.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:28 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/443999676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:30.004 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:29 smithi179 ceph-mon[25623]: from='client.15543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:30.004 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:29 smithi179 ceph-mon[25623]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:30.004 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:29 smithi179 ceph-mon[25623]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:30.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:29 smithi175 ceph-mon[29436]: from='client.15543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:30.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:29 smithi175 ceph-mon[29436]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:30.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:29 smithi175 ceph-mon[29436]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:29 smithi175 ceph-mon[24776]: from='client.15543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:29 smithi175 ceph-mon[24776]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:30.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:29 smithi175 ceph-mon[24776]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:21:30.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 systemd[1]: Stopping Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.458Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-25T06:21:30.768 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-25T06:21:30.769 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-25T06:21:30.769 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-25T06:21:30.769 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-25T06:21:30.769 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[60086]: ts=2024-05-25T06:21:30.459Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-25T06:21:30.769 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 podman[63624]: 2024-05-25 06:21:30.469219845 +0000 UTC m=+0.126292390 container died 254b9d2cd5d342a15cc65ba28d184cbc63383e94560b3bb9ed53d4e6a01c4546 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:31.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 podman[63624]: 2024-05-25 06:21:30.86728102 +0000 UTC m=+0.524353559 container cleanup 254b9d2cd5d342a15cc65ba28d184cbc63383e94560b3bb9ed53d4e6a01c4546 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:31.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:30 smithi179 bash[63624]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a 2024-05-25T06:21:31.482 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:31 smithi179 podman[63639]: 2024-05-25 06:21:31.127066461 +0000 UTC m=+0.656180319 container remove 254b9d2cd5d342a15cc65ba28d184cbc63383e94560b3bb9ed53d4e6a01c4546 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:31.482 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:31 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-25T06:21:31.482 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:31 smithi179 systemd[1]: Stopped Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:21:31.483 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:31 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service: Consumed 1.325s CPU time. 2024-05-25T06:21:31.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:31.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:31 smithi179 ceph-mon[25623]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:31.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:31 smithi179 systemd[1]: Starting Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:21:31.785 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:31 smithi179 podman[63769]: 2024-05-25 06:21:31.59976302 +0000 UTC m=+0.018344863 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-05-25T06:21:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:31 smithi175 ceph-mon[29436]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:31 smithi175 ceph-mon[24776]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:31 smithi179 podman[63769]: 2024-05-25 06:21:31.929239335 +0000 UTC m=+0.347821184 container create 5aff147d2abb5776e4cfb532300a5a357bf3b72976b763cc9ceb197b798be36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 podman[63769]: 2024-05-25 06:21:32.14019935 +0000 UTC m=+0.558781178 container init 5aff147d2abb5776e4cfb532300a5a357bf3b72976b763cc9ceb197b798be36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 podman[63769]: 2024-05-25 06:21:32.142958001 +0000 UTC m=+0.561539840 container start 5aff147d2abb5776e4cfb532300a5a357bf3b72976b763cc9ceb197b798be36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.167Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.167Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.167Z caller=main.go:570 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi179 (none))" 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.167Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-25T06:21:32.180 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.167Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-25T06:21:32.181 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.168Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-05-25T06:21:32.181 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.169Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-05-25T06:21:32.181 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.170Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-05-25T06:21:32.181 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.170Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-05-25T06:21:32.181 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.172Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-25T06:21:32.181 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.173Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=651.306µs 2024-05-25T06:21:32.181 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.173Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-25T06:21:32.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.180Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-05-25T06:21:32.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.194Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.204Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.207Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.210Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.211Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.211Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.211Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.097µs wal_replay_duration=38.379578ms wbl_replay_duration=111ns total_replay_duration=39.081574ms 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.215Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.215Z caller=main.go:1029 level=info msg="TSDB started" 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.215Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 bash[63769]: 5aff147d2abb5776e4cfb532300a5a357bf3b72976b763cc9ceb197b798be36e 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 systemd[1]: Started Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:21:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.236Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.417078ms db_storage=840ns remote_storage=1.183µs web_handler=341ns query_engine=686ns scrape=386.399µs scrape_sd=89.241µs notify=15.46µs notify_sd=15.29µs rules=20.611041ms tracing=6.387µs 2024-05-25T06:21:32.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.236Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-05-25T06:21:32.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:21:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:21:32.236Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-05-25T06:21:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:33 smithi179 ceph-mon[25623]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:33.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:33.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:33.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:33.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[29436]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:33.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:33.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:33.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:33.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:33 smithi175 ceph-mon[24776]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:35 smithi179 ceph-mon[25623]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:36.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:35 smithi175 ceph-mon[24776]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:36.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:35 smithi175 ceph-mon[29436]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:37 smithi179 ceph-mon[25623]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:37 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:37 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:37.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:37 smithi175 ceph-mon[29436]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:37 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:37 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:37 smithi175 ceph-mon[24776]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:21:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:37 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:37 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:38.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:38 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:38.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:38 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:38 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:38 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:38 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:38 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:39 smithi179 ceph-mon[25623]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:40.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:39 smithi175 ceph-mon[24776]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:40.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:39 smithi175 ceph-mon[29436]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:40.877 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:40 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:41.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:40 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:40 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:41 smithi179 ceph-mon[25623]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:42.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:41 smithi175 ceph-mon[24776]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:41 smithi175 ceph-mon[29436]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:43.258 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.258 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.258 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:43.258 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.259 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:43.260 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:43 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:21:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:21:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:21:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:43.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:43.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:21:43.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:43.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:21:43.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:21:43.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:43 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:21:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:21:43] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:44 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:21:44.133+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:44.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:21:44.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:44.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:44.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:44.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:44.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:44.327 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:44.327 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:44.327 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.511 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.512 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.512 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.512 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:44.512 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:44 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:21:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:45 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:46.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:46.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:46.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:46.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:46.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:46.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:45 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:21:46.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:45 smithi179 ceph-mon[25623]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:46.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:45 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:46.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:46.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:45 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:46.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:21:46.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:45 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:45 smithi175 ceph-mon[24776]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:45 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-25T06:21:46.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:45 smithi175 ceph-mon[29436]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:46 smithi179 ceph-mon[25623]: Upgrade: Updating alertmanager.a 2024-05-25T06:21:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:46 smithi179 ceph-mon[25623]: Deploying daemon alertmanager.a on smithi175 2024-05-25T06:21:47.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:46 smithi175 ceph-mon[24776]: Upgrade: Updating alertmanager.a 2024-05-25T06:21:47.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:46 smithi175 ceph-mon[24776]: Deploying daemon alertmanager.a on smithi175 2024-05-25T06:21:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:46 smithi175 ceph-mon[29436]: Upgrade: Updating alertmanager.a 2024-05-25T06:21:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:46 smithi175 ceph-mon[29436]: Deploying daemon alertmanager.a on smithi175 2024-05-25T06:21:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:47 smithi179 ceph-mon[25623]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 523 B/s rd, 0 op/s 2024-05-25T06:21:48.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:47 smithi175 ceph-mon[24776]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 523 B/s rd, 0 op/s 2024-05-25T06:21:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:47 smithi175 ceph-mon[29436]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 523 B/s rd, 0 op/s 2024-05-25T06:21:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:49 smithi179 ceph-mon[25623]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:50.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:49 smithi175 ceph-mon[24776]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:50.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:49 smithi175 ceph-mon[29436]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:51.701 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:51.701 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:51 smithi175 ceph-mon[24776]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:51.702 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:51.702 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:51 smithi175 ceph-mon[29436]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:21:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:51 smithi179 ceph-mon[25623]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:21:53.476 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:53 smithi175 systemd[1]: Stopping Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:21:53.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:53 smithi175 ceph-mon[29436]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:53 smithi175 ceph-mon[24776]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:53.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:21:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:21:53] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-05-25T06:21:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[58651]: level=info ts=2024-05-25T06:21:53.477Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:21:53.821 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:53 smithi175 podman[78032]: 2024-05-25 06:21:53.528241827 +0000 UTC m=+0.160958246 container died 1893f9da103a12c45b7bda5555f746a88f977f422999d56d7ff5e568e23b0635 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:54.015 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:53 smithi179 ceph-mon[25623]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:21:54.319 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:53 smithi175 podman[78032]: 2024-05-25 06:21:53.986535692 +0000 UTC m=+0.619252133 container cleanup 1893f9da103a12c45b7bda5555f746a88f977f422999d56d7ff5e568e23b0635 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:54.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:53 smithi175 bash[78032]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a 2024-05-25T06:21:54.574 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:54 smithi175 podman[78046]: 2024-05-25 06:21:54.480316183 +0000 UTC m=+0.944624647 container remove 1893f9da103a12c45b7bda5555f746a88f977f422999d56d7ff5e568e23b0635 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:54.574 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:54 smithi175 podman[78046]: 2024-05-25 06:21:54.555807311 +0000 UTC m=+1.020115776 volume remove 868acd48a0a36d04a77ffa65ba7e620368272fdc7c578262682c1f9a5d69864d 2024-05-25T06:21:54.975 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:54 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-25T06:21:54.976 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:54 smithi175 systemd[1]: Stopped Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:21:54.976 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:54 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service: Consumed 1.495s CPU time. 2024-05-25T06:21:55.273 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:54 smithi175 systemd[1]: Starting Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:21:55.274 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 podman[78186]: 2024-05-25 06:21:55.091467164 +0000 UTC m=+0.017602578 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-05-25T06:21:55.274 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 podman[78186]: 2024-05-25 06:21:55.190951885 +0000 UTC m=+0.117087330 volume create 1f9a54bdf8ee3ec615985e99315ccd1985c777a526df21de5d5afdee1e2b06c5 2024-05-25T06:21:55.274 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 podman[78186]: 2024-05-25 06:21:55.274677455 +0000 UTC m=+0.200812881 container create 12c8cd2eac3288ddd0c1080a93cb114c5d5daea9b2a7a569000caa88f52628d8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:55 smithi179 ceph-mon[25623]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:56.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-mon[24776]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:56.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:56.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:55 smithi175 ceph-mon[29436]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:56.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:21:56.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 podman[78186]: 2024-05-25 06:21:55.596355048 +0000 UTC m=+0.522490464 container init 12c8cd2eac3288ddd0c1080a93cb114c5d5daea9b2a7a569000caa88f52628d8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:56.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 podman[78186]: 2024-05-25 06:21:55.599009004 +0000 UTC m=+0.525144410 container start 12c8cd2eac3288ddd0c1080a93cb114c5d5daea9b2a7a569000caa88f52628d8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:21:56.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.610Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-05-25T06:21:56.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.610Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-05-25T06:21:56.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.612Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.175 port=9094 2024-05-25T06:21:56.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.613Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-25T06:21:56.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.638Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T06:21:56.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.638Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-25T06:21:56.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.641Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-05-25T06:21:56.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:55.641Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-05-25T06:21:56.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 bash[78186]: 12c8cd2eac3288ddd0c1080a93cb114c5d5daea9b2a7a569000caa88f52628d8 2024-05-25T06:21:56.071 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:55 smithi175 systemd[1]: Started Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:21:56.873 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:56 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:56.873 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:56 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:56.873 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:56 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:56.874 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:56 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:56.874 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:56 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:56.874 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:56 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:56 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:56 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:21:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:56 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:21:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:57 smithi179 ceph-mon[25623]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:58.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:57 smithi175 ceph-mon[24776]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:57 smithi175 ceph-mon[29436]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:21:58.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:21:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:21:57.614Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000342967s 2024-05-25T06:21:59.098 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:21:59.774 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:21:59.774 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 starting - - - - 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (14m) 23s ago 21m 55.9M - 8.3.5 dad864ee21e9 30f86eeb19f8 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 47s ago 21m 59.1M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (76s) 23s ago 26m 450M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (11m) 47s ago 28m 503M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (28m) 47s ago 28m 77.8M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (26m) 23s ago 26m 60.7M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (27m) 47s ago 27m 56.3M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (59s) 47s ago 22m 7960k - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (52s) 23s ago 22m 7884k - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (26m) 47s ago 26m 56.9M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (25m) 47s ago 25m 58.4M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (25m) 47s ago 25m 52.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:21:59.775 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (25m) 47s ago 25m 53.0M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:21:59.776 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (24m) 23s ago 24m 55.7M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:21:59.776 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (24m) 23s ago 24m 52.3M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:21:59.776 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (24m) 23s ago 24m 54.6M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:21:59.776 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (23m) 23s ago 23m 56.8M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:21:59.776 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (28s) 23s ago 22m 49.0M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:21:59.776 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (21m) 47s ago 21m 76.2M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:21:59.776 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (21m) 23s ago 21m 78.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:22:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:21:59 smithi179 ceph-mon[25623]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:00.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:21:59 smithi175 ceph-mon[29436]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:00.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:21:59 smithi175 ceph-mon[24776]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:00.207 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:22:00.208 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:22:00.208 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:22:00.208 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:00.208 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:22:00.208 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:22:00.209 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:00.209 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:22:00.209 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:22:00.209 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:00.209 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:22:00.209 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:22:00.210 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:22:00.210 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:00.210 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:22:00.210 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:22:00.210 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:22:00.211 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:22:00.211 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:22:00.553 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:22:00.553 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:22:00.553 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:22:00.553 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-25T06:22:00.554 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [ 2024-05-25T06:22:00.554 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr" 2024-05-25T06:22:00.554 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:22:00.554 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "2/2 daemons upgraded", 2024-05-25T06:22:00.554 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading alertmanager daemons", 2024-05-25T06:22:00.554 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:22:00.555 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:22:00.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='client.25453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/746047838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='client.25453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:00.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:00 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/746047838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='client.25453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:00 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/746047838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:01 smithi179 ceph-mon[25623]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:01 smithi179 ceph-mon[25623]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:01 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:01 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:02.042 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[24776]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[24776]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[29436]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[29436]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:02.043 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:01 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:03 smithi175 ceph-mon[24776]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:03.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:22:03] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-05-25T06:22:03.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:03 smithi175 ceph-mon[29436]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:04.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:03 smithi179 ceph-mon[25623]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:05.718 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:22:05.616Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00198893s 2024-05-25T06:22:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-05-25T06:22:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:06.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:05 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:06.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:06.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:06.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:05 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:22:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:06 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:06.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:22:06.634+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:06.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:06.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:06.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:06.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:06.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:06.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:06.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:06.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:06.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:06.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:06.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:06.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:06.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:06 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:22:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:07.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:06 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:07 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:08.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:08.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:08.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:08.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:07 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:22:08.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:07 smithi179 ceph-mon[25623]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:07 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:08.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:08.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:07 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:08.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:08.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:07 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:22:08.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:07 smithi175 ceph-mon[24776]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:08.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:08.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:08.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:08.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:08.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:08.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:07 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-25T06:22:08.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:07 smithi175 ceph-mon[29436]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:09 smithi179 ceph-mon[25623]: Upgrade: Updating grafana.a 2024-05-25T06:22:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:09 smithi179 ceph-mon[25623]: Deploying daemon grafana.a on smithi179 2024-05-25T06:22:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:09 smithi179 ceph-mon[25623]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:09 smithi175 ceph-mon[24776]: Upgrade: Updating grafana.a 2024-05-25T06:22:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:09 smithi175 ceph-mon[24776]: Deploying daemon grafana.a on smithi179 2024-05-25T06:22:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:09 smithi175 ceph-mon[24776]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:09 smithi175 ceph-mon[29436]: Upgrade: Updating grafana.a 2024-05-25T06:22:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:09 smithi175 ceph-mon[29436]: Deploying daemon grafana.a on smithi179 2024-05-25T06:22:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:09 smithi175 ceph-mon[29436]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:11.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:10 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:11.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:10 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:11.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:10 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:11 smithi175 ceph-mon[24776]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-05-25T06:22:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:11 smithi175 ceph-mon[29436]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-05-25T06:22:12.186 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:12.186 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:11 smithi179 ceph-mon[25623]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-05-25T06:22:13.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:13 smithi175 ceph-mon[24776]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:13 smithi175 ceph-mon[29436]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:13.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:22:13] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-05-25T06:22:14.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:13 smithi179 ceph-mon[25623]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:15.953 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:15 smithi179 ceph-mon[25623]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:22:16.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:15 smithi175 ceph-mon[24776]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:22:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:15 smithi175 ceph-mon[29436]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:22:18.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:17 smithi175 ceph-mon[29436]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:17 smithi175 ceph-mon[24776]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:18.078 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:17 smithi179 ceph-mon[25623]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:18.625 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:18 smithi179 systemd[1]: Stopping Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:22:18.893 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[49308]: t=2024-05-25T06:22:18+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-25T06:22:18.893 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:18 smithi179 podman[65825]: 2024-05-25 06:22:18.635859595 +0000 UTC m=+0.129311812 container died 30f86eeb19f8ff1417d05c56fa0b95eee0123d589b58e949d44292ccb9578336 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=236.1648460182, io.openshift.expose-services=, name=ubi8, architecture=x86_64, io.buildah.version=1.24.2, version=8.5, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-05-25T06:22:19.259 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:18 smithi179 podman[65825]: 2024-05-25 06:22:18.983949377 +0000 UTC m=+0.477401591 container cleanup 30f86eeb19f8ff1417d05c56fa0b95eee0123d589b58e949d44292ccb9578336 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, vcs-type=git, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, vendor=Red Hat, Inc., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8) 2024-05-25T06:22:19.259 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:18 smithi179 bash[65825]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a 2024-05-25T06:22:19.535 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:19 smithi179 podman[65837]: 2024-05-25 06:22:19.260070764 +0000 UTC m=+0.625421926 container remove 30f86eeb19f8ff1417d05c56fa0b95eee0123d589b58e949d44292ccb9578336 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, vcs-type=git, description=Ceph Grafana Container, maintainer=Paul Cuzner , io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, name=ubi8, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public) 2024-05-25T06:22:19.535 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:19 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-25T06:22:19.535 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:19 smithi179 systemd[1]: Stopped Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:22:19.535 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:19 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@grafana.a.service: Consumed 5.569s CPU time. 2024-05-25T06:22:19.946 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:19 smithi179 ceph-mon[25623]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:19.947 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:19 smithi179 systemd[1]: Starting Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:22:19.947 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:19 smithi179 podman[65969]: 2024-05-25 06:22:19.726377664 +0000 UTC m=+0.017944820 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-05-25T06:22:19.947 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:19 smithi179 podman[65969]: 2024-05-25 06:22:19.828006238 +0000 UTC m=+0.119573370 container create fb91afb62378e4509d748d48238025bcf84de4ef3961ae5d642952906365574f (image=quay.io/ceph/grafana:9.4.12, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-25T06:22:20.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:19 smithi175 ceph-mon[24776]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:20.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:19 smithi175 ceph-mon[29436]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:20.205 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 podman[65969]: 2024-05-25 06:22:20.055708083 +0000 UTC m=+0.347275215 container init fb91afb62378e4509d748d48238025bcf84de4ef3961ae5d642952906365574f (image=quay.io/ceph/grafana:9.4.12, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-25T06:22:20.205 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 podman[65969]: 2024-05-25 06:22:20.058623498 +0000 UTC m=+0.350190639 container start fb91afb62378e4509d748d48238025bcf84de4ef3961ae5d642952906365574f (image=quay.io/ceph/grafana:9.4.12, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-25T06:22:20.205 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 bash[65969]: fb91afb62378e4509d748d48238025bcf84de4ef3961ae5d642952906365574f 2024-05-25T06:22:20.205 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 systemd[1]: Started Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:22:20.535 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.205959027Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206212779Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206265243Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206294318Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206301898Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206306392Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206310743Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206315206Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206319562Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206323967Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206328137Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-25T06:22:20.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.20633239Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-25T06:22:20.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206340389Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-25T06:22:20.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206345441Z level=info msg="Path Home" path=/usr/share/grafana 2024-05-25T06:22:20.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206349989Z level=info msg="Path Data" path=/var/lib/grafana 2024-05-25T06:22:20.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206354057Z level=info msg="Path Logs" path=/var/log/grafana 2024-05-25T06:22:20.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206357779Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-05-25T06:22:20.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206361438Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=settings t=2024-05-25T06:22:20.206365319Z level=info msg="App mode production" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=sqlstore t=2024-05-25T06:22:20.206427826Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=sqlstore t=2024-05-25T06:22:20.206445448Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.218987812Z level=info msg="Starting DB migrations" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.222287938Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.23346335Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.234683586Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.235576219Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.236846426Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.239761516Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.242490181Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-05-25T06:22:20.538 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.245371124Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.248031542Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.248569095Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.248928195Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.25000009Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.25033102Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.251408237Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.252505601Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.253643944Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.253957424Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.25504495Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.256135978Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.256487918Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-05-25T06:22:20.539 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.257691722Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.258822949Z level=info msg="Executing migration" id="create provenance_type table" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.259289397Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.262151587Z level=info msg="Executing migration" id="create alert_image table" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.264886152Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.265424587Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.265747674Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.266174554Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.266639507Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.266730825Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.266979755Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.267493749Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-05-25T06:22:20.540 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.267939789Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.268665848Z level=info msg="Executing migration" id="create secrets table" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.269093344Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.27537365Z level=info msg="Executing migration" id="add name column into data_keys" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.276509895Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.276835217Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.283109668Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.289571393Z level=info msg="Executing migration" id="add column hidden to role table" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.290707933Z level=info msg="Executing migration" id="create query_history table v1" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.291443534Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.292153872Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-05-25T06:22:20.541 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.292855335Z level=info msg="Executing migration" id="teams permissions migration" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.293627771Z level=info msg="Executing migration" id="dashboard permissions" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.294999256Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.2958305Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.296617492Z level=info msg="Executing migration" id="alerting notification permissions" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.296994605Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.297429451Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.297908051Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-05-25T06:22:20.542 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.299063016Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.299787486Z level=info msg="Executing migration" id="create correlation table v1" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.300554612Z level=info msg="Executing migration" id="add index correlations.uid" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.301030192Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.301812203Z level=info msg="Executing migration" id="add correlation config column" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.302966419Z level=info msg="Executing migration" id="create entity_events table" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.303732924Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.304171527Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.304263005Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.304926872Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-25T06:22:20.543 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.30501744Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.305743051Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.306567463Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.307002935Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.309639401Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.312360951Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.314977449Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.315433343Z level=info msg="Executing migration" id="Drop public config table" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.318113742Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.320856806Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.323247244Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.323723538Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.326348493Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.33021477Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-05-25T06:22:20.544 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.332877372Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.334062978Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.336632485Z level=info msg="Executing migration" id="add share column" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.339402198Z level=info msg="Executing migration" id="create default alerting folders" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.342073084Z level=info msg="Executing migration" id="create file table" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.342575562Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.345309972Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.347867286Z level=info msg="Executing migration" id="create file_meta table" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.350135499Z level=info msg="Executing migration" id="file table idx: path key" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.350577615Z level=info msg="Executing migration" id="set path collation in file table" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.35301773Z level=info msg="Executing migration" id="managed permissions migration" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.35413058Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.355467439Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-05-25T06:22:20.545 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.357942697Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.359218762Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.361621805Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.363995995Z level=info msg="Executing migration" id="update group index for alert rules" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.366546043Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.367159123Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.369521281Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.372004103Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.3746444Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.377096843Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.390444817Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.391176318Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.391893759Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-05-25T06:22:20.546 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.396360349Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.39706522Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.397805663Z level=info msg="Executing migration" id="create folder table" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.398459808Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.398916505Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.399403109Z level=info msg="Executing migration" id="Update folder title length" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.400101594Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=migrator t=2024-05-25T06:22:20.400831435Z level=info msg="migrations completed" performed=105 skipped=376 duration=178.582268ms 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=sqlstore t=2024-05-25T06:22:20.401174534Z level=info msg="Created default organization" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=plugin.finder t=2024-05-25T06:22:20.433709075Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=secrets t=2024-05-25T06:22:20.433815014Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=query_data t=2024-05-25T06:22:20.437736424Z level=info msg="Query Service initialization" 2024-05-25T06:22:20.547 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=live.push_http t=2024-05-25T06:22:20.440602551Z level=info msg="Live Push Gateway initialization" 2024-05-25T06:22:21.471 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:21.471 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:21.471 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:21.472 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:21.472 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[29436]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:21.472 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:21.472 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:21.472 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:21.472 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:21.472 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:21 smithi175 ceph-mon[24776]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:21 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:21 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:21 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:21 smithi179 ceph-mon[25623]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=infra.usagestats.collector t=2024-05-25T06:22:21.134476237Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=provisioning.datasources t=2024-05-25T06:22:21.136084637Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=provisioning.datasources t=2024-05-25T06:22:21.136093587Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=provisioning.datasources t=2024-05-25T06:22:21.136203289Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=provisioning.alerting t=2024-05-25T06:22:21.158308411Z level=info msg="starting to provision alerting" 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=provisioning.alerting t=2024-05-25T06:22:21.158323385Z level=info msg="finished to provision alerting" 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=grafanaStorageLogger t=2024-05-25T06:22:21.158633245Z level=info msg="storage starting" 2024-05-25T06:22:21.536 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=ngalert.state.manager t=2024-05-25T06:22:21.158709365Z level=info msg="Warming state cache for startup" 2024-05-25T06:22:21.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=http.server t=2024-05-25T06:22:21.161383749Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-05-25T06:22:21.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=ngalert.state.manager t=2024-05-25T06:22:21.179480804Z level=info msg="State cache has been initialized" states=0 duration=20.769374ms 2024-05-25T06:22:21.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=ticker t=2024-05-25T06:22:21.179542832Z level=info msg=starting first_tick=2024-05-25T06:22:30Z 2024-05-25T06:22:21.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=ngalert.multiorg.alertmanager t=2024-05-25T06:22:21.179544119Z level=info msg="starting MultiOrg Alertmanager" 2024-05-25T06:22:21.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=sqlstore.transactions t=2024-05-25T06:22:21.200256231Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-05-25T06:22:21.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=sqlstore.transactions t=2024-05-25T06:22:21.21102539Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-05-25T06:22:21.537 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:22:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=sqlstore.transactions t=2024-05-25T06:22:21.222012195Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-05-25T06:22:23.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:22:23] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-05-25T06:22:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:23 smithi175 ceph-mon[29436]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:23 smithi175 ceph-mon[24776]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:23 smithi179 ceph-mon[25623]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:25 smithi179 ceph-mon[25623]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:25 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:26.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:25 smithi175 ceph-mon[29436]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:26.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:25 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:25 smithi175 ceph-mon[24776]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:25 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:28.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:27 smithi175 ceph-mon[29436]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:28.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:27 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:28.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:27 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:27 smithi175 ceph-mon[24776]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:27 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:27 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:28.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:27 smithi179 ceph-mon[25623]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:28.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:27 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:28.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:27 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:29 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:29 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:29 smithi179 ceph-mon[25623]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:29.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:29 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:29 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:29 smithi175 ceph-mon[24776]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:29 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:29 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:29 smithi175 ceph-mon[29436]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:30.917 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:22:31.560 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:22:31.560 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (36s) 31s ago 22m 17.4M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:22:31.560 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (11s) 4s ago 22m 51.6M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:22:31.560 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 31s ago 21m 61.2M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:22:31.560 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (108s) 4s ago 27m 451M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (11m) 31s ago 28m 507M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (28m) 31s ago 28m 79.6M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (27m) 4s ago 27m 56.1M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (27m) 31s ago 27m 56.7M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (90s) 31s ago 23m 8262k - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (84s) 4s ago 23m 15.2M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (26m) 31s ago 26m 56.8M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (26m) 31s ago 26m 58.5M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (26m) 31s ago 26m 52.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (25m) 31s ago 25m 53.0M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (25m) 4s ago 25m 55.7M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:22:31.561 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (24m) 4s ago 24m 52.3M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:22:31.562 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (24m) 4s ago 24m 54.6M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:22:31.562 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (24m) 4s ago 24m 56.8M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:22:31.562 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (59s) 4s ago 23m 56.3M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:22:31.562 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (21m) 31s ago 21m 76.4M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:22:31.562 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (21m) 4s ago 21m 79.1M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:22:31.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:31 smithi175 ceph-mon[29436]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:31.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:31.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:31 smithi175 ceph-mon[24776]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:31.964 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:22:31.964 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:22:31.964 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:22:31.964 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:22:31.965 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:22:31.966 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:22:31.966 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:22:31.966 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:22:32.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:32.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:31 smithi179 ceph-mon[25623]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:32.307 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:22:32.307 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:22:32.307 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:22:32.307 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-25T06:22:32.307 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [ 2024-05-25T06:22:32.307 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr" 2024-05-25T06:22:32.308 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:22:32.308 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "2/2 daemons upgraded", 2024-05-25T06:22:32.308 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading grafana daemons", 2024-05-25T06:22:32.308 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:22:32.308 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:22:32.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='client.25471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:32.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:32.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:32.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:32.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/909974342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='client.25471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:32.976 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/909974342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:32.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:32 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='client.25471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/909974342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:33.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:32 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:22:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:22:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:22:33.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:33.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:22:33.825 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:22:33] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:22:33.733+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:33.826 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:33.827 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:33.828 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:33.829 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:33.830 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:33.830 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:33.830 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:33.830 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:33.830 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:22:34.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:22:34.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:22:34.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:22:34.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:22:34.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:22:34.899 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:34.900 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:34.901 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:34.901 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:34.901 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:34.901 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:34.901 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:34.901 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:34.901 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:34.902 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:34.902 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:34.902 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:34.902 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.902 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.902 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:34.903 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.904 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:34.905 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:34.906 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:34.907 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:34.908 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:34.909 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: Upgrade: Finalizing container_image settings 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:22:34.910 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.911 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:34.912 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: Upgrade: Finalizing container_image settings 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:22:34.913 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:22:34.914 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.914 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.914 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:22:34.914 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:22:34.914 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.915 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:34.916 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:34.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:34.919 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:34.920 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:34.920 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:34 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:22:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:35.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: Upgrade: Finalizing container_image settings 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:22:35.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:22:35.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:22:35.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:35.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:34 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:36.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[24776]: Upgrade: Complete! 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[24776]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[24776]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 721 B/s rd, 0 op/s 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[29436]: Upgrade: Complete! 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[29436]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:22:36.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[29436]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 721 B/s rd, 0 op/s 2024-05-25T06:22:36.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:36.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:36.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:35 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:36.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:22:35.795+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:36.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:36.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:36.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:36.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:36.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:36.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:36.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:35 smithi179 ceph-mon[25623]: Upgrade: Complete! 2024-05-25T06:22:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:35 smithi179 ceph-mon[25623]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:22:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:35 smithi179 ceph-mon[25623]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 721 B/s rd, 0 op/s 2024-05-25T06:22:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:35 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:35 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:35 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:37.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:37.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:37.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:37.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-05-25T06:22:37.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1009 B/s rd, 0 op/s 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:37.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:37.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-05-25T06:22:37.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1009 B/s rd, 0 op/s 2024-05-25T06:22:37.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:37.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:37.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:37.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:36 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:37.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1009 B/s rd, 0 op/s 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:22:37.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:36 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:22:37.808 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:22:37.665+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.808 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.809 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.810 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.810 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.810 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.810 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:37.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:37.812 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:37.813 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:37.814 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:37.814 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:37 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:38.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:37 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:38.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:37 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:37 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:37 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:37 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:22:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:37 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:38 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:39.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:39.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:38 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:39.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:39.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:38 smithi175 ceph-mon[24776]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:39.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:38 smithi175 ceph-mon[29436]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-05-25T06:22:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:38 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:22:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:22:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:22:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:22:39.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:38 smithi179 ceph-mon[25623]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-05-25T06:22:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:40 smithi179 ceph-mon[25623]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 649 B/s rd, 0 op/s 2024-05-25T06:22:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:40 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:41.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:40 smithi175 ceph-mon[29436]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 649 B/s rd, 0 op/s 2024-05-25T06:22:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:40 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:40 smithi175 ceph-mon[24776]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 649 B/s rd, 0 op/s 2024-05-25T06:22:41.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:40 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:42.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:41 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:42.952 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:42 smithi179 ceph-mon[25623]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:22:43.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:42 smithi175 ceph-mon[24776]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:22:43.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:42 smithi175 ceph-mon[29436]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:22:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:22:43] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.43.0" 2024-05-25T06:22:44.121 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:43 smithi179 ceph-mon[25623]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:22:44.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:43 smithi175 ceph-mon[24776]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:22:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:43 smithi175 ceph-mon[29436]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:22:46.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:45 smithi179 ceph-mon[25623]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:22:46.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:45 smithi175 ceph-mon[24776]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:22:46.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:45 smithi175 ceph-mon[29436]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:22:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:47 smithi179 ceph-mon[25623]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:48.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:47 smithi175 ceph-mon[29436]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:47 smithi175 ceph-mon[24776]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:50 smithi179 ceph-mon[25623]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:51.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:50 smithi175 ceph-mon[24776]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:50 smithi175 ceph-mon[29436]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:51 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:22:53.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:53 smithi175 ceph-mon[24776]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:53 smithi175 ceph-mon[29436]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:53.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:22:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:22:53] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.43.0" 2024-05-25T06:22:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:53 smithi179 ceph-mon[25623]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:22:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:55 smithi179 ceph-mon[25623]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:55 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:56.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:55 smithi175 ceph-mon[29436]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:56.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:56.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:55 smithi175 ceph-mon[24776]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:56.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:22:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:57 smithi179 ceph-mon[25623]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:58.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:57 smithi175 ceph-mon[24776]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:22:58.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:57 smithi175 ceph-mon[29436]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:22:59 smithi179 ceph-mon[25623]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:00.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:22:59 smithi175 ceph-mon[24776]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:22:59 smithi175 ceph-mon[29436]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:01 smithi179 ceph-mon[25623]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:01 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:02.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:01 smithi175 ceph-mon[24776]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:02.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:01 smithi175 ceph-mon[29436]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:02.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:03.241 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-25T06:23:03.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:03 smithi175 ceph-mon[29436]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:03.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:03 smithi175 ceph-mon[29436]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:03 smithi175 ceph-mon[24776]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:03.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:03 smithi175 ceph-mon[24776]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:03.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:23:03] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.43.0" 2024-05-25T06:23:04.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:03 smithi179 ceph-mon[25623]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:04.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:03 smithi179 ceph-mon[25623]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:05.291 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:23:05.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:05 smithi175 ceph-mon[29436]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:05 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2768802999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:05 smithi175 ceph-mon[24776]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:05 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2768802999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:05.975 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-05-25T06:23:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:05 smithi179 ceph-mon[25623]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:05 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/2768802999' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:07 smithi175 ceph-mon[29436]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:07.849 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:07 smithi175 ceph-mon[24776]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:08.004 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)" 2024-05-25T06:23:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:07 smithi179 ceph-mon[25623]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:08.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-05-25T06:23:08.916 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:08 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3821710929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:08.917 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:08 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3821710929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:08 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3821710929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:09.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:09 smithi175 ceph-mon[24776]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:09.959 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:09 smithi175 ceph-mon[29436]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:09 smithi179 ceph-mon[25623]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:23:10.922 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:10 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:10.922 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:10 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/3130500448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:10.922 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:10 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:10.922 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:10 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/3130500448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:10 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:10 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/3130500448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:11.221 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-05-25T06:23:11.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:11 smithi175 ceph-mon[29436]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:11.774 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:11.774 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:11 smithi175 ceph-mon[24776]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:11.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:11 smithi179 ceph-mon[25623]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:11 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:13.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:13 smithi175 ceph-mon[24776]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:13 smithi175 ceph-mon[29436]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:13.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:23:13] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.43.0" 2024-05-25T06:23:14.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:13 smithi179 ceph-mon[25623]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:14 smithi179 ceph-mon[25623]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:15.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:14 smithi175 ceph-mon[24776]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:14 smithi175 ceph-mon[29436]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:15.250 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:23:15.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:15 smithi175 ceph-mon[24776]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:15 smithi175 ceph-mon[29436]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:15.880 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:23:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:15 smithi179 ceph-mon[25623]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": null, 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": false, 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "", 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": null, 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "", 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:23:17.869 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:23:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:17 smithi179 ceph-mon[25623]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:18.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:17 smithi175 ceph-mon[24776]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:17 smithi175 ceph-mon[29436]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:18.416 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:23:19.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:19 smithi175 ceph-mon[24776]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:19.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:19 smithi175 ceph-mon[24776]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:19 smithi175 ceph-mon[29436]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:19 smithi175 ceph-mon[29436]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:19 smithi179 ceph-mon[25623]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:20.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:19 smithi179 ceph-mon[25623]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:20.635 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:23:20.635 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:23:20.635 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:23:20.931 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:20 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/108053733' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:23:20.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:20 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/108053733' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:23:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:20 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/108053733' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:23:21.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-05-25T06:23:21.777 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:21 smithi175 ceph-mon[24776]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:21.777 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:21 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:21.777 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:21 smithi175 ceph-mon[29436]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:21.777 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:21 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:21 smithi179 ceph-mon[25623]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:22.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:21 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:23.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:23 smithi175 ceph-mon[29436]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:23 smithi175 ceph-mon[24776]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:23.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:23:23] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.43.0" 2024-05-25T06:23:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:23 smithi179 ceph-mon[25623]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:24.923 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:24 smithi175 ceph-mon[29436]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:24.923 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:24 smithi175 ceph-mon[29436]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mon", "hosts": "smithi179", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:24.924 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:24 smithi175 ceph-mon[24776]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:24.924 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:24 smithi175 ceph-mon[24776]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mon", "hosts": "smithi179", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:24 smithi179 ceph-mon[25623]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:24 smithi179 ceph-mon[25623]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mon", "hosts": "smithi179", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:25.683 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:23:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:25 smithi179 ceph-mon[25623]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:25 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:25 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:26.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:25 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[29436]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[24776]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:26.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:25 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:26.404 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[29436]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:26.980 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[24776]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:23:26.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:26.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:26.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:26.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:26.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:26.981 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:26 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:26 smithi179 ceph-mon[25623]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:23:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:26 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:26 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:26 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:27 smithi179 ceph-mon[25623]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:27 smithi179 ceph-mon[25623]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:23:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:27 smithi179 ceph-mon[25623]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:27 smithi179 ceph-mon[25623]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:27 smithi179 ceph-mon[25623]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[24776]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[24776]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:23:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[24776]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:28.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:28.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[29436]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:28.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[29436]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:23:28.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[29436]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:28.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:28.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:27 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:29.135 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:23:29.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:23:29.312+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:29.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:29.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:29.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:23:29.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:29.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:29.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:29.820 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:23:29.820 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (94s) 89s ago 23m 17.4M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:23:29.820 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (70s) 62s ago 23m 51.6M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:23:29.820 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 89s ago 22m 61.2M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:23:29.820 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (2m) 62s ago 28m 451M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (12m) 89s ago 29m 507M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (29m) 89s ago 29m 79.6M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (28m) 62s ago 28m 56.1M 2048M 17.2.0 e1d6a67b021e bc465d7efed0 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (28m) 89s ago 28m 56.7M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (2m) 89s ago 24m 8262k - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (2m) 62s ago 24m 15.2M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (27m) 89s ago 27m 56.8M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (27m) 89s ago 27m 58.5M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (27m) 89s ago 27m 52.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (26m) 89s ago 26m 53.0M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (26m) 62s ago 26m 55.7M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (25m) 62s ago 25m 52.3M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (25m) 62s ago 25m 54.6M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:23:29.821 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (25m) 62s ago 25m 56.8M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:23:29.822 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (118s) 62s ago 24m 56.3M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:23:29.822 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (22m) 89s ago 22m 76.4M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:23:29.822 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (22m) 62s ago 22m 79.1M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:23:30.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:29 smithi175 ceph-mon[24776]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:30.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:29 smithi175 ceph-mon[29436]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:30.222 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:23:30.223 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:23:30.224 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:23:30.224 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:23:30.224 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:23:30.224 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-25T06:23:30.224 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:23:30.224 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:23:30.224 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:23:30.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:29 smithi179 ceph-mon[25623]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:30.577 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi179", 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "", 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 image", 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:23:30.578 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[24776]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:31.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:31.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[29436]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:31.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:31.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:31.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:31.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:31.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[24776]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[24776]: from='client.15681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[24776]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1015930518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[29436]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:23:31.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[29436]: from='client.15681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[29436]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:30 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1015930518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:30 smithi179 ceph-mon[25623]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:30 smithi179 ceph-mon[25623]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:31.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:31.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:31.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:31.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:31.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:31.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:31.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:31.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:31.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:31.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:31.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:31.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:31.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:23:31.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:31.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:30 smithi179 ceph-mon[25623]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:30 smithi179 ceph-mon[25623]: from='client.15681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:30 smithi179 ceph-mon[25623]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:31.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:30 smithi179 ceph-mon[25623]: from='client.? 172.21.15.175:0/1015930518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='client.25579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:23:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='client.25579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:23:32.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:31 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-25T06:23:32.101 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:32.101 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='client.25579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:23:32.101 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-25T06:23:32.101 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:32.101 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:32.101 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:32.102 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:32.102 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:23:32.102 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:31 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-25T06:23:33.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[24776]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:23:33.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[24776]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:23:33.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all mgr 2024-05-25T06:23:33.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[24776]: Upgrade: It appears safe to stop mon.b 2024-05-25T06:23:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[29436]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:23:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[29436]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:23:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all mgr 2024-05-25T06:23:33.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:32 smithi175 ceph-mon[29436]: Upgrade: It appears safe to stop mon.b 2024-05-25T06:23:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:32 smithi179 ceph-mon[25623]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:23:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:32 smithi179 ceph-mon[25623]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:23:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:32 smithi179 ceph-mon[25623]: Upgrade: Setting container_image for all mgr 2024-05-25T06:23:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:32 smithi179 ceph-mon[25623]: Upgrade: It appears safe to stop mon.b 2024-05-25T06:23:33.799 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:23:33] "GET /metrics HTTP/1.1" 200 38001 "" "Prometheus/2.43.0" 2024-05-25T06:23:33.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:33 smithi179 ceph-mon[25623]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:33.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:33 smithi179 ceph-mon[25623]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:33.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:23:33.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:23:33.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:33 smithi179 ceph-mon[25623]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:34.068 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[24776]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:34.068 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:34.068 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:23:34.068 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:23:34.068 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:34.068 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[29436]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:34.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:34.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:23:34.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:23:34.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:33 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:35.018 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:34 smithi179 systemd[1]: Stopping Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:23:35.018 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:34 smithi179 ceph-mon[25623]: Upgrade: Updating mon.b 2024-05-25T06:23:35.018 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:34 smithi179 ceph-mon[25623]: Deploying daemon mon.b on smithi179 2024-05-25T06:23:35.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:34 smithi175 ceph-mon[24776]: Upgrade: Updating mon.b 2024-05-25T06:23:35.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:34 smithi175 ceph-mon[24776]: Deploying daemon mon.b on smithi179 2024-05-25T06:23:35.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:34 smithi175 ceph-mon[29436]: Upgrade: Updating mon.b 2024-05-25T06:23:35.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:34 smithi175 ceph-mon[29436]: Deploying daemon mon.b on smithi179 2024-05-25T06:23:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b[25619]: 2024-05-25T06:23:35.017+0000 7f22cde36700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:23:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b[25619]: 2024-05-25T06:23:35.017+0000 7f22cde36700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-05-25T06:23:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 podman[67419]: 2024-05-25 06:23:35.038895534 +0000 UTC m=+0.123768071 container died bc465d7efed04e0ab8a9849ab7d054c29f4b6fb932a87499592006dd6645c991 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, ceph=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-25T06:23:35.677 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 podman[67419]: 2024-05-25 06:23:35.409621263 +0000 UTC m=+0.494493784 container cleanup bc465d7efed04e0ab8a9849ab7d054c29f4b6fb932a87499592006dd6645c991 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, maintainer=Guillaume Abrioux , io.openshift.expose-services=, name=centos-stream, distribution-scope=public, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-25T06:23:35.678 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 bash[67419]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b 2024-05-25T06:23:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 podman[67433]: 2024-05-25 06:23:35.678258662 +0000 UTC m=+0.633462776 container remove bc465d7efed04e0ab8a9849ab7d054c29f4b6fb932a87499592006dd6645c991 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, RELEASE=HEAD, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-25T06:23:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.b.service: Deactivated successfully. 2024-05-25T06:23:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 systemd[1]: Stopped Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:23:36.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:35 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.b.service: Consumed 17.175s CPU time. 2024-05-25T06:23:36.385 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 systemd[1]: Starting Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:23:36.386 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 podman[67561]: 2024-05-25 06:23:36.157201937 +0000 UTC m=+0.017128571 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:23:36.386 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 podman[67561]: 2024-05-25 06:23:36.262744317 +0000 UTC m=+0.122670937 container create 8cb1d1e3627b934755c29dac51d56b6fe9212ea8d5123847d638ce87c71099b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2) 2024-05-25T06:23:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 podman[67561]: 2024-05-25 06:23:36.524001861 +0000 UTC m=+0.383928482 container init 8cb1d1e3627b934755c29dac51d56b6fe9212ea8d5123847d638ce87c71099b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 podman[67561]: 2024-05-25 06:23:36.527189801 +0000 UTC m=+0.387116424 container start 8cb1d1e3627b934755c29dac51d56b6fe9212ea8d5123847d638ce87c71099b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c) 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: set uid:gid to 167:167 (ceph:ceph) 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc), process ceph-mon, pid 2 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: pidfile_write: ignore empty --pid-file 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: load: jerasure load: lrc 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: RocksDB version: 7.9.2 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Git sha 0 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Compile date 2024-05-23 20:42:41 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: DB SUMMARY 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: DB Session ID: XRZ17U48WS9OG5SK9Y4S 2024-05-25T06:23:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: CURRENT file: CURRENT 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: IDENTITY file: IDENTITY 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2843 Bytes 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 3627765 ; 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.error_if_exists: 0 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.create_if_missing: 0 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.paranoid_checks: 1 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-25T06:23:36.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.env: 0x563edd6efc60 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.fs: PosixFileSystem 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.info_log: 0x563eded35b20 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_file_opening_threads: 16 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.statistics: (nil) 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.use_fsync: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_log_file_size: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.keep_log_file_num: 1000 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.recycle_log_file_num: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.allow_fallocate: 1 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.allow_mmap_reads: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.allow_mmap_writes: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.use_direct_reads: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-25T06:23:36.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.create_missing_column_families: 0 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.db_log_dir: 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.wal_dir: 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.advise_random_on_open: 1 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.db_write_buffer_size: 0 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.write_buffer_manager: 0x563eded06820 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-25T06:23:36.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.rate_limiter: (nil) 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.wal_recovery_mode: 2 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.enable_thread_tracking: 0 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.enable_pipelined_write: 0 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.unordered_write: 0 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.row_cache: None 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.wal_filter: None 2024-05-25T06:23:36.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-25T06:23:36.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.allow_ingest_behind: 0 2024-05-25T06:23:36.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.two_write_queues: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.manual_wal_flush: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.wal_compression: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.atomic_flush: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.log_readahead_size: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.best_efforts_recovery: 0 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-25T06:23:36.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.allow_data_in_errors: 0 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.db_host_id: __hostname__ 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_background_jobs: 2 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_background_compactions: -1 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_subcompactions: 1 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_total_wal_size: 0 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-25T06:23:36.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_open_files: -1 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bytes_per_sync: 0 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_readahead_size: 0 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_background_flushes: -1 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Compression algorithms supported: 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kZSTD supported: 0 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kXpressCompression supported: 0 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kLZ4HCCompression supported: 1 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kZlibCompression supported: 1 2024-05-25T06:23:36.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kSnappyCompression supported: 1 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kLZ4Compression supported: 1 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: kBZip2Compression supported: 0 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.merge_operator: 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_filter: None 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_filter_factory: None 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.sst_partitioner_factory: None 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-25T06:23:36.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563eded35c40) 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks: 1 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_top_level_index_and_filter: 1 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_type: 0 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_index_type: 0 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_shortening: 1 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: checksum: 4 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: no_block_cache: 0 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache: 0x563eded37090 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_name: BinnedLRUCache 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_options: 2024-05-25T06:23:36.796 INFO:journalctl@ceph.mon.b.smithi179.stdout: capacity : 536870912 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: num_shard_bits : 4 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: strict_capacity_limit : 0 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: high_pri_pool_ratio: 0.000 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_compressed: (nil) 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: persistent_cache: (nil) 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size: 4096 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size_deviation: 10 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_restart_interval: 16 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_block_restart_interval: 1 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: metadata_block_size: 4096 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: partition_filters: 0 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_delta_encoding: 1 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: filter_policy: bloomfilter 2024-05-25T06:23:36.797 INFO:journalctl@ceph.mon.b.smithi179.stdout: whole_key_filtering: 1 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: verify_compression: 0 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: read_amp_bytes_per_bit: 0 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: format_version: 5 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_index_compression: 1 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_align: 0 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: max_auto_readahead_size: 262144 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: prepopulate_block_cache: 0 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: initial_auto_readahead_size: 8192 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout: num_file_reads_for_auto_readahead: 2 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.write_buffer_size: 33554432 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_write_buffer_number: 2 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression: NoCompression 2024-05-25T06:23:36.798 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression: Disabled 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.prefix_extractor: nullptr 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.num_levels: 7 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-25T06:23:36.799 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.level: 32767 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.strategy: 0 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.enabled: false 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-25T06:23:36.800 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.target_file_size_base: 67108864 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-25T06:23:36.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.arena_block_size: 1048576 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.disable_auto_compactions: 0 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-25T06:23:36.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.table_properties_collectors: 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.inplace_update_support: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.bloom_locality: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.max_successive_merges: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.paranoid_file_checks: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.force_consistency_checks: 1 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.report_bg_io_stats: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.ttl: 2592000 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-25T06:23:36.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.enable_blob_files: false 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.min_blob_size: 0 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.blob_file_size: 268435456 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.blob_file_starting_level: 0 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28666, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-05-25T06:23:36.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ec3f1c41-8be2-45f9-b4c9-9bf357f96ea3 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618216556671, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618216572873, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 2251307, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28671, "largest_seqno": 30729, "table_properties": {"data_size": 2243292, "index_size": 5011, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1989, "raw_key_size": 18539, "raw_average_key_size": 23, "raw_value_size": 2226747, "raw_average_value_size": 2840, "num_data_blocks": 224, "num_entries": 784, "num_filter_entries": 784, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716618216, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ec3f1c41-8be2-45f9-b4c9-9bf357f96ea3", "db_session_id": "XRZ17U48WS9OG5SK9Y4S", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618216572989, "job": 1, "event": "recovery_finished"} 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563edee26000 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: DB pointer 0x563edee10000 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** DB Stats ** 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T06:23:36.805 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: L0 1/0 2.15 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 196.0 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: L6 1/0 8.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: Sum 2/0 11.00 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 196.0 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 196.0 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T06:23:36.806 INFO:journalctl@ceph.mon.b.smithi179.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 196.0 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative compaction: 0.00 GB write, 107.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval compaction: 0.00 GB write, 107.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Block cache BinnedLRUCache@0x563eded37090#2 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.2e-05 secs_since: 0 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-05-25T06:23:36.807 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: starting mon.b rank 2 at public addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] at bind addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???) e3 preinit fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???).mds e1 new map 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???).mds e1 print_map 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout: e1 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout: legacy client fscid: -1 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout: No filesystems configured 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???).osd e107 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:23:36.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:23:36.809 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-05-25T06:23:36.809 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 bash[67561]: 8cb1d1e3627b934755c29dac51d56b6fe9212ea8d5123847d638ce87c71099b8 2024-05-25T06:23:36.809 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 systemd[1]: Started Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:23:37.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:36 smithi175 ceph-mon[24776]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:36 smithi175 ceph-mon[29436]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:36 smithi179 ceph-mon[67580]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:23:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: mon.b calling monitor election 2024-05-25T06:23:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: mon.c calling monitor election 2024-05-25T06:23:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: mon.a calling monitor election 2024-05-25T06:23:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:23:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: monmap e3: 3 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T06:23:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: fsmap 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: mgrmap e39: y(active, since 3m), standbys: x 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:38.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:37 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: mon.b calling monitor election 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: mon.c calling monitor election 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: mon.a calling monitor election 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: monmap e3: 3 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: fsmap 2024-05-25T06:23:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: mgrmap e39: y(active, since 3m), standbys: x 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: mon.b calling monitor election 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: mon.c calling monitor election 2024-05-25T06:23:38.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: mon.a calling monitor election 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: monmap e3: 3 mons at {a=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0]} 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: fsmap 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: mgrmap e39: y(active, since 3m), standbys: x 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:38.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:37 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:40.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:39 smithi179 ceph-mon[67580]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:40.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:39 smithi175 ceph-mon[24776]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:40.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:39 smithi175 ceph-mon[29436]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:41.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:40 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:41.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:40 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:41.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:40 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:42.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:41 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:23:41.582+0000 7f7fc9ec7700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-05-25T06:23:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[29436]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:42.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:42.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[24776]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:42.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:42.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:41 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:41 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:41 smithi179 ceph-mon[67580]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:41 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:41 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:43.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:42 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:43.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:42 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:43.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:42 smithi179 ceph-mon[67580]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:43.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:42 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:42 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:42 smithi175 ceph-mon[24776]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:42 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:42 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:42 smithi175 ceph-mon[29436]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:23:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:23:43] "GET /metrics HTTP/1.1" 200 37943 "" "Prometheus/2.43.0" 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:45 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:46.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:45 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:46 smithi179 ceph-mon[67580]: Detected new or changed devices on smithi179 2024-05-25T06:23:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:46 smithi179 ceph-mon[67580]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-05-25T06:23:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:46 smithi179 ceph-mon[67580]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-05-25T06:23:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:46 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:47.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:46 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[24776]: Detected new or changed devices on smithi179 2024-05-25T06:23:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[24776]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-05-25T06:23:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[24776]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-05-25T06:23:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:47.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi179 2024-05-25T06:23:47.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[29436]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-05-25T06:23:47.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[29436]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-05-25T06:23:47.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:47.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:46 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:23:47.197+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:47.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:47.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:47.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:47.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:47.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:47.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:47.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:23:48.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:23:48.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:48.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:48.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:48.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:47 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:48.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:23:48.077 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:23:48.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:23:48.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:23:48.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:48.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.083 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.084 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:48.085 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:48.086 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:48.086 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:48.086 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:48.086 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:47 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:23:49.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all crash 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mds 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:23:49.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nfs 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all node-exporter 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all prometheus 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all alertmanager 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all grafana 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all loki 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all promtail 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Finalizing container_image settings 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Upgrade: Complete! 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:49.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:48 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:23:49.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:49.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.046 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:49.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.050 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all crash 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all mds 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all nfs 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all node-exporter 2024-05-25T06:23:49.051 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all prometheus 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all alertmanager 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all grafana 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all loki 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all promtail 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Finalizing container_image settings 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Upgrade: Complete! 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:49.052 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all crash 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all mds 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all nfs 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all node-exporter 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all prometheus 2024-05-25T06:23:49.053 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all alertmanager 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all grafana 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all loki 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all promtail 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Finalizing container_image settings 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Upgrade: Complete! 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:49.054 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:48 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:23:49.185+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:49.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:49.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:49.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:49.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:49.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:49.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:49.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:49.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:49 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:50.023 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:50.024 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:50.025 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:50.025 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:49 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:49 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:23:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:49 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:49 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:23:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:49 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:49 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:49 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:23:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:49 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:23:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:50 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:51.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:50 smithi179 ceph-mon[67580]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 777 B/s rd, 0 op/s 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:50 smithi179 ceph-mon[67580]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:50 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:50 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:51.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:50 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:51.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:51.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:51.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.080 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[24776]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 777 B/s rd, 0 op/s 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[24776]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.081 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[29436]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 777 B/s rd, 0 op/s 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[29436]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:50 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:23:51.082 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:23:50.899+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:51.084 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:51.085 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:51.086 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:51.087 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:51.087 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:51 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:52.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:51 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:51 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:23:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:52 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:52.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:52.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:52 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:52.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:52.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:52.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:52.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:52.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:52.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:52.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:52.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:52.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:52 smithi175 ceph-mon[24776]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-05-25T06:23:52.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:52.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:52.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:52.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:52.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:52.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:52 smithi175 ceph-mon[29436]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-05-25T06:23:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:52 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:23:53.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:53.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:23:53.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:23:53.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:23:53.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:52 smithi179 ceph-mon[67580]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-05-25T06:23:53.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:23:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:23:53] "GET /metrics HTTP/1.1" 200 37943 "" "Prometheus/2.43.0" 2024-05-25T06:23:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:54 smithi179 ceph-mon[67580]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:23:55.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:54 smithi175 ceph-mon[24776]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:23:55.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:54 smithi175 ceph-mon[29436]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:23:56.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:55 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:56.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:55 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:56.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:55 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:23:57.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:56 smithi179 ceph-mon[67580]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-05-25T06:23:57.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:56 smithi175 ceph-mon[24776]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-05-25T06:23:57.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:56 smithi175 ceph-mon[29436]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-05-25T06:23:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:23:58 smithi179 ceph-mon[67580]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:23:59.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:23:58 smithi175 ceph-mon[24776]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:23:59.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:23:58 smithi175 ceph-mon[29436]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:00 smithi179 ceph-mon[67580]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:24:01.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:00 smithi175 ceph-mon[24776]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:24:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:00 smithi175 ceph-mon[29436]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:24:01.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:24:01.977 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:01 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:01.977 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:01 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:02.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:01 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:02.958 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:02 smithi175 ceph-mon[29436]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:02.959 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:02 smithi175 ceph-mon[29436]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:02.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:02 smithi175 ceph-mon[24776]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:02.959 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:02 smithi175 ceph-mon[24776]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:02 smithi179 ceph-mon[67580]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:03.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:02 smithi179 ceph-mon[67580]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:03.692 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:24:03.692 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (2m) 2m ago 24m 17.4M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (103s) 22s ago 23m 78.7M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 2m ago 23m 61.2M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (3m) 22s ago 28m 452M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (13m) 2m ago 30m 507M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (30m) 2m ago 30m 79.6M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (27s) 22s ago 29m 19.1M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (29m) 2m ago 29m 56.7M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (3m) 2m ago 24m 8262k - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (2m) 22s ago 24m 14.6M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (28m) 2m ago 28m 56.8M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (28m) 2m ago 28m 58.5M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (27m) 2m ago 27m 52.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:24:03.693 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (27m) 2m ago 27m 53.0M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:24:03.694 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (26m) 22s ago 26m 56.0M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:24:03.694 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (26m) 22s ago 26m 52.7M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:24:03.694 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (26m) 22s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:24:03.694 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (25m) 22s ago 25m 57.1M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:24:03.694 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (2m) 22s ago 25m 68.2M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:24:03.694 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (23m) 2m ago 23m 76.4M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:24:03.694 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (23m) 22s ago 23m 79.6M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:24:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:24:03] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-05-25T06:24:04.300 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-05-25T06:24:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:04 smithi175 ceph-mon[24776]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:04 smithi175 ceph-mon[24776]: from='client.25591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:04 smithi175 ceph-mon[29436]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:04 smithi175 ceph-mon[29436]: from='client.25591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:04 smithi179 ceph-mon[67580]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:05.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:04 smithi179 ceph-mon[67580]: from='client.25591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:06.363 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:24:06.964 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:06 smithi175 ceph-mon[29436]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:06.964 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:06 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/503071076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:06.964 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:06 smithi175 ceph-mon[24776]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:06.964 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:06 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/503071076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:06.990 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:24:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:06 smithi179 ceph-mon[67580]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:06 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/503071076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:08 smithi175 ceph-mon[24776]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:08 smithi175 ceph-mon[29436]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:08.975 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": null, 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": false, 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "", 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": null, 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "", 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:24:08.976 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:24:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:08 smithi179 ceph-mon[67580]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:09.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:24:11.021 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:10 smithi175 ceph-mon[29436]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:11.021 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:10 smithi175 ceph-mon[29436]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:11.021 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:10 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:11.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:10 smithi175 ceph-mon[24776]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:11.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:10 smithi175 ceph-mon[24776]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:11.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:10 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:10 smithi179 ceph-mon[67580]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:10 smithi179 ceph-mon[67580]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:10 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:11.685 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:24:11.686 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:24:11.686 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:24:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:11 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:11 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2447660511' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:24:12.045 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:11 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:12.045 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:11 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/2447660511' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:24:12.045 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:11 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:12.045 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:11 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/2447660511' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:24:12.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-05-25T06:24:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:12 smithi179 ceph-mon[67580]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:13.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:12 smithi175 ceph-mon[24776]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:13.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:12 smithi175 ceph-mon[29436]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:13.723 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:24:13] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.43.0" 2024-05-25T06:24:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:14 smithi179 ceph-mon[67580]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:15.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:14 smithi175 ceph-mon[24776]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:14 smithi175 ceph-mon[29436]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:24:15.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:15 smithi175 ceph-mon[24776]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:15 smithi175 ceph-mon[24776]: from='client.25621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mon", "hosts": "smithi175", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:15 smithi175 ceph-mon[29436]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:15 smithi175 ceph-mon[29436]: from='client.25621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mon", "hosts": "smithi175", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:15 smithi179 ceph-mon[67580]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:15 smithi179 ceph-mon[67580]: from='client.25621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "mon", "hosts": "smithi175", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:16.795 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:24:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:16 smithi179 ceph-mon[67580]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:17.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:16 smithi175 ceph-mon[24776]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:17.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:16 smithi175 ceph-mon[29436]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:24:17.571 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-25T06:24:17.816 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:17.817 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:17.818 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:24:17.818 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:17.818 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:17.818 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:17.818 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:24:17.818 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:24:17.818 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:17.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:17.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:17.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:17.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:17 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:18.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:24:18.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:18.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:18.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:18.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:24:18.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:24:18.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:18.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:18.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:18.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:18.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:17 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:18.975 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:18 smithi175 ceph-mon[24776]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:24:18.975 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:18 smithi175 ceph-mon[24776]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:24:18.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:18 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:24:18.975 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:18 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:24:18.975 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:18.797+0000 7f7fc6ec1700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:18.975 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:18.976 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:18.977 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:18.978 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:18.979 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:18.980 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:18.980 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:18.980 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:18.980 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:18.980 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:18.980 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:18.980 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:18 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:19.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:18 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:24:19.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:18 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:24:20.040 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:19 smithi175 ceph-mon[24776]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:20.041 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:20.042 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:20.043 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:20.044 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:19 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:20.045 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:20.046 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:20.047 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:20.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:19 smithi175 ceph-mon[24776]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:19 smithi175 ceph-mon[24776]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:19 smithi175 ceph-mon[29436]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:24:20.049 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:19 smithi175 ceph-mon[29436]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:19 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:20.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:20.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:20.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:20.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:20.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:19 smithi179 ceph-mon[67580]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:24:20.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:19 smithi179 ceph-mon[67580]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:20.754 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:24:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:20 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:20 smithi179 ceph-mon[67580]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:20 smithi179 ceph-mon[67580]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:21.317 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:20 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:21.317 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:20 smithi175 ceph-mon[29436]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:21.317 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:20 smithi175 ceph-mon[29436]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:21.318 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:20 smithi175 ceph-mon[24776]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:21.318 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:20 smithi175 ceph-mon[24776]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:21.318 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:20 smithi175 ceph-mon[24776]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:21.432 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:24:21.432 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (2m) 2m ago 24m 17.4M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:24:21.432 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (2m) 40s ago 24m 78.7M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:24:21.432 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 2m ago 23m 61.2M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:24:21.432 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (3m) 40s ago 29m 452M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (13m) 2m ago 30m 507M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (30m) 2m ago 30m 79.6M 2048M 17.2.0 e1d6a67b021e b34a4a648ba0 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (45s) 40s ago 29m 19.1M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (29m) 2m ago 29m 56.7M 2048M 17.2.0 e1d6a67b021e ea2c474b542a 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (3m) 2m ago 25m 8262k - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (3m) 40s ago 25m 14.6M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (28m) 2m ago 28m 56.8M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (28m) 2m ago 28m 58.5M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (27m) 2m ago 27m 52.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (27m) 2m ago 27m 53.0M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (27m) 40s ago 27m 56.0M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (26m) 40s ago 26m 52.7M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (26m) 40s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:24:21.433 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (26m) 40s ago 25m 57.1M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:24:21.434 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (2m) 40s ago 25m 68.2M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:24:21.434 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (23m) 2m ago 23m 76.4M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:24:21.434 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (23m) 40s ago 23m 79.6M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:24:21.839 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:24:21.839 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:24:21.839 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-25T06:24:21.839 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 1 2024-05-25T06:24:21.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:21.839 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "mds": {}, 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:24:21.840 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:24:22.192 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:24:22.192 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:24:22.192 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:24:22.192 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi175", 2024-05-25T06:24:22.192 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:24:22.193 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "0/2 daemons upgraded", 2024-05-25T06:24:22.193 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading mon daemons", 2024-05-25T06:24:22.193 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:24:22.193 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:24:22.811 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='client.15759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: Upgrade: Setting container_image for all mgr 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: Upgrade: It appears safe to stop mon.a 2024-05-25T06:24:22.812 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[29436]: from='client.? 172.21.15.175:0/1804008646' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='client.15759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: Upgrade: Setting container_image for all mgr 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: Upgrade: It appears safe to stop mon.a 2024-05-25T06:24:22.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:22 smithi175 ceph-mon[24776]: from='client.? 172.21.15.175:0/1804008646' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='client.15759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:24:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:24:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:23.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:23.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mgr 2024-05-25T06:24:23.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:23.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:24:23.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-25T06:24:23.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: Upgrade: It appears safe to stop mon.a 2024-05-25T06:24:23.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:22 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1804008646' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:24:23] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.43.0" 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[24776]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[24776]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[24776]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[24776]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[29436]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[29436]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:24:23.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:24:23.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:23 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:23 smithi179 ceph-mon[67580]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:24:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:23 smithi179 ceph-mon[67580]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:24:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:23 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='mgr.y' 2024-05-25T06:24:24.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:23 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:24:24.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:23 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:24:24.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:23 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:24.799 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:24 smithi175 ceph-mon[29436]: Upgrade: Updating mon.a 2024-05-25T06:24:24.799 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:24 smithi175 ceph-mon[29436]: Deploying daemon mon.a on smithi175 2024-05-25T06:24:24.799 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:24 smithi175 ceph-mon[24776]: Upgrade: Updating mon.a 2024-05-25T06:24:24.799 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:24 smithi175 ceph-mon[24776]: Deploying daemon mon.a on smithi175 2024-05-25T06:24:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:24 smithi179 ceph-mon[67580]: Upgrade: Updating mon.a 2024-05-25T06:24:25.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:24 smithi179 ceph-mon[67580]: Deploying daemon mon.a on smithi175 2024-05-25T06:24:25.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:25 smithi175 systemd[1]: Stopping Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:24:25.544 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a[24772]: 2024-05-25T06:24:25.319+0000 7eff79b63700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:24:25.545 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a[24772]: 2024-05-25T06:24:25.319+0000 7eff79b63700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-05-25T06:24:25.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:25 smithi175 podman[84491]: 2024-05-25 06:24:25.5448663 +0000 UTC m=+0.317514874 container died b34a4a648ba02b1e6c165033a6f89f34f239deb13bf69d4277c4adfc977ad085 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, release=754, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.buildah.version=1.19.8, vendor=Red Hat, Inc., version=8) 2024-05-25T06:24:26.128 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:25 smithi175 podman[84491]: 2024-05-25 06:24:25.911167531 +0000 UTC m=+0.683816089 container cleanup b34a4a648ba02b1e6c165033a6f89f34f239deb13bf69d4277c4adfc977ad085 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_BRANCH=HEAD, name=centos-stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc.) 2024-05-25T06:24:26.128 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:25 smithi175 bash[84491]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a 2024-05-25T06:24:26.469 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 podman[84505]: 2024-05-25 06:24:26.12919765 +0000 UTC m=+0.578158452 container remove b34a4a648ba02b1e6c165033a6f89f34f239deb13bf69d4277c4adfc977ad085 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, ceph=True, architecture=x86_64, version=8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, name=centos-stream, release=754, io.openshift.expose-services=, RELEASE=HEAD) 2024-05-25T06:24:26.469 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-25T06:24:26.469 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 systemd[1]: Stopped Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:24:26.470 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service: Consumed 25.813s CPU time. 2024-05-25T06:24:26.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 systemd[1]: Starting Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:24:26.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 podman[84630]: 2024-05-25 06:24:26.588100328 +0000 UTC m=+0.017891535 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:24:26.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 podman[84630]: 2024-05-25 06:24:26.714095697 +0000 UTC m=+0.143886905 container create 57a27054d700ee77e14d2992c9cf2ad901bd143253efaad849743bda58d2c15a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:24:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 podman[84630]: 2024-05-25 06:24:26.924832294 +0000 UTC m=+0.354623493 container init 57a27054d700ee77e14d2992c9cf2ad901bd143253efaad849743bda58d2c15a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-05-25T06:24:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 podman[84630]: 2024-05-25 06:24:26.928195103 +0000 UTC m=+0.357986290 container start 57a27054d700ee77e14d2992c9cf2ad901bd143253efaad849743bda58d2c15a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-05-25T06:24:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: set uid:gid to 167:167 (ceph:ceph) 2024-05-25T06:24:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc), process ceph-mon, pid 2 2024-05-25T06:24:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: pidfile_write: ignore empty --pid-file 2024-05-25T06:24:27.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: load: jerasure load: lrc 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: RocksDB version: 7.9.2 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Git sha 0 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Compile date 2024-05-23 20:42:41 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: DB SUMMARY 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: DB Session ID: DK01HZ1Z1NBEWWP4JF31 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: CURRENT file: CURRENT 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: IDENTITY file: IDENTITY 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3109 Bytes 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 721894 ; 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.error_if_exists: 0 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.create_if_missing: 0 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.paranoid_checks: 1 2024-05-25T06:24:27.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-25T06:24:27.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-25T06:24:27.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-25T06:24:27.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.env: 0x561c88747c60 2024-05-25T06:24:27.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.fs: PosixFileSystem 2024-05-25T06:24:27.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.info_log: 0x561c8987db20 2024-05-25T06:24:27.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_file_opening_threads: 16 2024-05-25T06:24:27.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.statistics: (nil) 2024-05-25T06:24:27.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.use_fsync: 0 2024-05-25T06:24:27.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_log_file_size: 0 2024-05-25T06:24:27.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.keep_log_file_num: 1000 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.recycle_log_file_num: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.allow_fallocate: 1 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.allow_mmap_reads: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.allow_mmap_writes: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.use_direct_reads: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.create_missing_column_families: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.db_log_dir: 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.wal_dir: 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-25T06:24:27.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.advise_random_on_open: 1 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.db_write_buffer_size: 0 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.write_buffer_manager: 0x561c8984e820 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.rate_limiter: (nil) 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.wal_recovery_mode: 2 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.enable_thread_tracking: 0 2024-05-25T06:24:27.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.enable_pipelined_write: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.unordered_write: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.row_cache: None 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.wal_filter: None 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.allow_ingest_behind: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.two_write_queues: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.manual_wal_flush: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.wal_compression: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.atomic_flush: 0 2024-05-25T06:24:27.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.log_readahead_size: 0 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.best_efforts_recovery: 0 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.allow_data_in_errors: 0 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.db_host_id: __hostname__ 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_background_jobs: 2 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_background_compactions: -1 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_subcompactions: 1 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-25T06:24:27.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_total_wal_size: 0 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_open_files: -1 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bytes_per_sync: 0 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_readahead_size: 0 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_background_flushes: -1 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Compression algorithms supported: 2024-05-25T06:24:27.328 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kZSTD supported: 0 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kXpressCompression supported: 0 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kLZ4HCCompression supported: 1 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kZlibCompression supported: 1 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kSnappyCompression supported: 1 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kLZ4Compression supported: 1 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: kBZip2Compression supported: 0 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.merge_operator: 2024-05-25T06:24:27.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_filter: None 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_filter_factory: None 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.sst_partitioner_factory: None 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561c8987dc40) 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: cache_index_and_filter_blocks: 1 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: pin_top_level_index_and_filter: 1 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: index_type: 0 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: data_block_index_type: 0 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: index_shortening: 1 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-25T06:24:27.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: checksum: 4 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: no_block_cache: 0 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache: 0x561c8987f090 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache_name: BinnedLRUCache 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache_options: 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: capacity : 536870912 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: num_shard_bits : 4 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: strict_capacity_limit : 0 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: high_pri_pool_ratio: 0.000 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_cache_compressed: (nil) 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: persistent_cache: (nil) 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_size: 4096 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_size_deviation: 10 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_restart_interval: 16 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: index_block_restart_interval: 1 2024-05-25T06:24:27.331 INFO:journalctl@ceph.mon.a.smithi175.stdout: metadata_block_size: 4096 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: partition_filters: 0 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_delta_encoding: 1 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: filter_policy: bloomfilter 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: whole_key_filtering: 1 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: verify_compression: 0 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: read_amp_bytes_per_bit: 0 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: format_version: 5 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: enable_index_compression: 1 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: block_align: 0 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: max_auto_readahead_size: 262144 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: prepopulate_block_cache: 0 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: initial_auto_readahead_size: 8192 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout: num_file_reads_for_auto_readahead: 2 2024-05-25T06:24:27.332 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.write_buffer_size: 33554432 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_write_buffer_number: 2 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression: NoCompression 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression: Disabled 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.prefix_extractor: nullptr 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.num_levels: 7 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-25T06:24:27.333 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.level: 32767 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.strategy: 0 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.enabled: false 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-25T06:24:27.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.target_file_size_base: 67108864 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-25T06:24:27.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.arena_block_size: 1048576 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.disable_auto_compactions: 0 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-25T06:24:27.336 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.table_properties_collectors: 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.inplace_update_support: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.bloom_locality: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.max_successive_merges: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.paranoid_file_checks: 0 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.force_consistency_checks: 1 2024-05-25T06:24:27.337 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.report_bg_io_stats: 0 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.ttl: 2592000 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.enable_blob_files: false 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.min_blob_size: 0 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.blob_file_size: 268435456 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.blob_file_starting_level: 0 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-25T06:24:27.338 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29128, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 4a43ed62-fd34-4c5b-89c2-98de36120ae1 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618266960049, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618266965026, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 706806, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29119, "largest_seqno": 29853, "table_properties": {"data_size": 703129, "index_size": 1703, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 965, "raw_key_size": 9868, "raw_average_key_size": 26, "raw_value_size": 695539, "raw_average_value_size": 1840, "num_data_blocks": 75, "num_entries": 378, "num_filter_entries": 378, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716618266, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "4a43ed62-fd34-4c5b-89c2-98de36120ae1", "db_session_id": "DK01HZ1Z1NBEWWP4JF31", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618266965141, "job": 1, "event": "recovery_finished"} 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561c8996e000 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: DB pointer 0x561c89958000 2024-05-25T06:24:27.339 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** DB Stats ** 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: L0 1/0 690.24 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 235.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:27.340 INFO:journalctl@ceph.mon.a.smithi175.stdout: L6 1/0 9.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: Sum 2/0 10.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 235.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 235.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 235.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-25T06:24:27.341 INFO:journalctl@ceph.mon.a.smithi175.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: Cumulative compaction: 0.00 GB write, 88.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: Interval compaction: 0.00 GB write, 88.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: Block cache BinnedLRUCache@0x561c8987f090#2 capacity: 512.00 MB usage: 2.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: starting mon.a rank 0 at public addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] at bind addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???) e3 preinit fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???).mds e1 new map 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???).mds e1 print_map 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: e1 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-25T06:24:27.342 INFO:journalctl@ceph.mon.a.smithi175.stdout: legacy client fscid: -1 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout: 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout: No filesystems configured 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???).osd e107 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:26 smithi175 ceph-mon[84648]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:27 smithi175 bash[84630]: 57a27054d700ee77e14d2992c9cf2ad901bd143253efaad849743bda58d2c15a 2024-05-25T06:24:27.343 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:27 smithi175 systemd[1]: Started Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:24:28.782 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: ignoring --setuser ceph since I am not root 2024-05-25T06:24:28.782 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: ignoring --setgroup ceph since I am not root 2024-05-25T06:24:28.782 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mgr[60585]: -- 172.21.15.179:0/1083353023 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556ce80b25a0 con 0x556ce8086800 2024-05-25T06:24:28.782 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:28.595+0000 7f7a07db4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:28.782+0000 7f7a07db4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: mon.a calling monitor election 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: monmap epoch 3 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: last_changed 2024-05-25T05:55:03.364297+0000 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: min_mon_release 17 (quincy) 2024-05-25T06:24:28.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: election_strategy: 1 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: fsmap 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: mgrmap e39: y(active, since 4m), standbys: x 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: from='mgr.15387 ' entity='' 2024-05-25T06:24:28.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:28 smithi179 ceph-mon[67580]: mgrmap e40: y(active, since 4m), standbys: x 2024-05-25T06:24:28.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ignoring --setuser ceph since I am not root 2024-05-25T06:24:28.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ignoring --setgroup ceph since I am not root 2024-05-25T06:24:28.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mgr[63288]: -- 172.21.15.175:0/4166380937 <== mon.1 v2:172.21.15.175:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5613310f05a0 con 0x5613310c4c00 2024-05-25T06:24:28.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:28.595+0000 7f7f22110200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-25T06:24:28.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:28.782+0000 7f7f22110200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:24:28.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:28.785 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: mon.a calling monitor election 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: monmap epoch 3 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: last_changed 2024-05-25T05:55:03.364297+0000 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: min_mon_release 17 (quincy) 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: election_strategy: 1 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: fsmap 2024-05-25T06:24:28.786 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: mgrmap e39: y(active, since 4m), standbys: x 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: from='mgr.15387 ' entity='' 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[84648]: mgrmap e40: y(active, since 4m), standbys: x 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: from='mgr.15387 172.21.15.175:0/3619342570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: mon.a calling monitor election 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:24:28.787 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: monmap epoch 3 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: last_changed 2024-05-25T05:55:03.364297+0000 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: min_mon_release 17 (quincy) 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: election_strategy: 1 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: fsmap 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: osdmap e107: 8 total, 8 up, 8 in 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: mgrmap e39: y(active, since 4m), standbys: x 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:24:28.788 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:24:28.789 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:24:28.789 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: from='mgr.15387 ' entity='' 2024-05-25T06:24:28.789 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:28 smithi175 ceph-mon[29436]: mgrmap e40: y(active, since 4m), standbys: x 2024-05-25T06:24:29.035 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:28.885+0000 7f7a07db4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:24:29.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:28.896+0000 7f7f22110200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-25T06:24:29.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:29.196+0000 7f7a07db4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:24:29.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:29.282+0000 7f7a07db4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:24:29.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:29.208+0000 7f7f22110200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-25T06:24:29.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:29 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:29.296+0000 7f7f22110200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-25T06:24:30.660 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:30 smithi175 ceph-mon[29436]: mgrmap e41: y(active, since 4m), standbys: x 2024-05-25T06:24:30.660 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:30 smithi175 ceph-mon[84648]: mgrmap e41: y(active, since 4m), standbys: x 2024-05-25T06:24:30.660 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:30.487+0000 7f7f22110200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:24:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:30 smithi179 ceph-mon[67580]: mgrmap e41: y(active, since 4m), standbys: x 2024-05-25T06:24:30.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:30.475+0000 7f7a07db4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-25T06:24:30.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:30.647+0000 7f7a07db4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:24:31.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:30.659+0000 7f7f22110200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-25T06:24:31.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:30.833+0000 7f7f22110200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:24:31.112 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:30.821+0000 7f7a07db4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-25T06:24:31.414 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:31.112+0000 7f7a07db4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:24:31.415 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:31.206+0000 7f7a07db4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:24:31.415 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:31.414+0000 7f7a07db4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:24:31.425 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:31 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:31.125+0000 7f7f22110200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-25T06:24:31.425 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:31 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:31.219+0000 7f7f22110200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-25T06:24:31.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:31.611+0000 7f7a07db4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:24:31.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:31 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:31.425+0000 7f7f22110200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-25T06:24:31.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:31 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:31.622+0000 7f7f22110200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-25T06:24:32.535 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:32.221+0000 7f7a07db4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:24:32.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:24:32.220Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:24:32.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:24:32.225Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:24:32.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:24:32.225Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:24:32.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:24:32.225Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:24:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:24:32.225Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:24:32.569 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:32 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:32.236+0000 7f7f22110200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-25T06:24:32.964 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:32.656+0000 7f7a07db4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:24:32.964 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:32.756+0000 7f7a07db4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:24:32.964 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:32.963+0000 7f7a07db4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:24:32.989 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:32 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:32.671+0000 7f7f22110200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-25T06:24:32.990 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:32 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:32.770+0000 7f7f22110200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-25T06:24:33.319 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:32 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:32.989+0000 7f7f22110200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-25T06:24:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:33.506+0000 7f7a07db4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:24:33.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:33.602+0000 7f7a07db4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:24:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:33.524+0000 7f7f22110200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-25T06:24:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:33.621+0000 7f7f22110200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-25T06:24:34.209 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:33.853+0000 7f7a07db4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:24:34.209 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:33.990+0000 7f7a07db4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:24:34.209 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:34 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:34.089+0000 7f7a07db4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:24:34.209 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:34 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:34.208+0000 7f7a07db4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:24:34.238 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:33.878+0000 7f7f22110200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-25T06:24:34.238 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:34.016+0000 7f7f22110200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-25T06:24:34.238 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:34.115+0000 7f7f22110200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-25T06:24:34.547 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:34.238+0000 7f7f22110200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-25T06:24:34.548 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:24:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:24:34.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:24:34.548 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:24:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:24:34.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:24:34.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:34 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:34.517+0000 7f7a07db4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:24:34.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:34.547+0000 7f7f22110200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-25T06:24:35.412 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:35.130+0000 7f7a07db4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:24:35.412 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:35.358+0000 7f7a07db4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:24:35.412 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: [25/May/2024:06:24:35] ENGINE Bus STARTING 2024-05-25T06:24:35.485 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:35.165+0000 7f7f22110200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-25T06:24:35.485 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:35.394+0000 7f7f22110200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-25T06:24:35.485 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:35] ENGINE Bus STARTING 2024-05-25T06:24:35.485 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: Standby manager daemon x restarted 2024-05-25T06:24:35.485 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: Standby manager daemon x started 2024-05-25T06:24:35.485 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: Active manager daemon y restarted 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: Activating manager daemon y 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: mgrmap e42: y(active, starting, since 0.00666786s), standbys: x 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:24:35.486 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: Standby manager daemon x restarted 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: Standby manager daemon x started 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: Active manager daemon y restarted 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: Activating manager daemon y 2024-05-25T06:24:35.487 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: mgrmap e42: y(active, starting, since 0.00666786s), standbys: x 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:24:35.488 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:24:35.744 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CherryPy Checker: 2024-05-25T06:24:35.744 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: The Application mounted at '' has an empty config. 2024-05-25T06:24:35.744 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:35.744 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:35] ENGINE Serving on http://:::9283 2024-05-25T06:24:35.744 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:35] ENGINE Bus STARTED 2024-05-25T06:24:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: CherryPy Checker: 2024-05-25T06:24:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: The Application mounted at '' has an empty config. 2024-05-25T06:24:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: 2024-05-25T06:24:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: [25/May/2024:06:24:35] ENGINE Serving on http://:::9283 2024-05-25T06:24:35.785 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:24:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x[60581]: [25/May/2024:06:24:35] ENGINE Bus STARTED 2024-05-25T06:24:35.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: Standby manager daemon x restarted 2024-05-25T06:24:35.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: Standby manager daemon x started 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.? 172.21.15.179:0/2613168981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: Active manager daemon y restarted 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: Activating manager daemon y 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: mgrmap e42: y(active, starting, since 0.00666786s), standbys: x 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-25T06:24:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-25T06:24:35.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:24:35.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:24:36.070 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:24:35 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:24:35.744Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=3 2024-05-25T06:24:36.693 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:24:36.438Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-05-25T06:24:36.694 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:36.429+0000 7f7edf5ab700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-05-25T06:24:36.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:24:36.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:24:36.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:24:36.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:24:36.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: Manager daemon y is now available 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:24:36.695 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: Manager daemon y is now available 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:24:36.696 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:24:36.697 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:36 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:24:36.777 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: Manager daemon y is now available 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:36.778 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:36.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:24:36.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-25T06:24:36.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:24:36.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:36 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[29436]: [25/May/2024:06:24:35] ENGINE Bus STARTING 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[29436]: [25/May/2024:06:24:35] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[29436]: [25/May/2024:06:24:36] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[29436]: [25/May/2024:06:24:36] ENGINE Bus STARTED 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[29436]: mgrmap e43: y(active, since 1.03164s), standbys: x 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[29436]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[84648]: [25/May/2024:06:24:35] ENGINE Bus STARTING 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[84648]: [25/May/2024:06:24:35] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:24:37.689 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[84648]: [25/May/2024:06:24:36] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:24:37.690 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[84648]: [25/May/2024:06:24:36] ENGINE Bus STARTED 2024-05-25T06:24:37.690 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[84648]: mgrmap e43: y(active, since 1.03164s), standbys: x 2024-05-25T06:24:37.690 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:37.690 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:37 smithi175 ceph-mon[84648]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:37 smithi179 ceph-mon[67580]: [25/May/2024:06:24:35] ENGINE Bus STARTING 2024-05-25T06:24:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:37 smithi179 ceph-mon[67580]: [25/May/2024:06:24:35] ENGINE Serving on http://172.21.15.175:8765 2024-05-25T06:24:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:37 smithi179 ceph-mon[67580]: [25/May/2024:06:24:36] ENGINE Serving on https://172.21.15.175:7150 2024-05-25T06:24:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:37 smithi179 ceph-mon[67580]: [25/May/2024:06:24:36] ENGINE Bus STARTED 2024-05-25T06:24:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:37 smithi179 ceph-mon[67580]: mgrmap e43: y(active, since 1.03164s), standbys: x 2024-05-25T06:24:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:37 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:37 smithi179 ceph-mon[67580]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:38 smithi179 ceph-mon[67580]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:38 smithi179 ceph-mon[67580]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:24:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:38 smithi179 ceph-mon[67580]: Cluster is now healthy 2024-05-25T06:24:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:38 smithi179 ceph-mon[67580]: mgrmap e44: y(active, since 2s), standbys: x 2024-05-25T06:24:38.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[29436]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:38.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[29436]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:24:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[29436]: Cluster is now healthy 2024-05-25T06:24:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[29436]: mgrmap e44: y(active, since 2s), standbys: x 2024-05-25T06:24:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[84648]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[84648]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:24:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[84648]: Cluster is now healthy 2024-05-25T06:24:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:38 smithi175 ceph-mon[84648]: mgrmap e44: y(active, since 2s), standbys: x 2024-05-25T06:24:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:40 smithi179 ceph-mon[67580]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:40 smithi179 ceph-mon[67580]: mgrmap e45: y(active, since 4s), standbys: x 2024-05-25T06:24:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:40 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:40 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:40 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:40 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[84648]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[84648]: mgrmap e45: y(active, since 4s), standbys: x 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[29436]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[29436]: mgrmap e45: y(active, since 4s), standbys: x 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:40.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:40 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[29436]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.228 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.229 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.229 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:42 smithi175 ceph-mon[84648]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:42 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:42 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:42 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:42 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:42 smithi179 ceph-mon[67580]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:24:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:24:43] "GET /metrics HTTP/1.1" 200 35212 "" "Prometheus/2.43.0" 2024-05-25T06:24:44.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:44 smithi175 ceph-mon[84648]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:24:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:44 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:44 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:44 smithi175 ceph-mon[29436]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:24:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:44 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:44 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:44 smithi179 ceph-mon[67580]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-25T06:24:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:44 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:44 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: Detected new or changed devices on smithi175 2024-05-25T06:24:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:24:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:24:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:24:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:24:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:24:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:24:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:24:46.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:45 smithi179 ceph-mon[67580]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: Detected new or changed devices on smithi175 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:24:46.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[29436]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: Detected new or changed devices on smithi175 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: Updating smithi175:/etc/ceph/ceph.conf 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: Updating smithi179:/etc/ceph/ceph.conf 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:24:46.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.conf 2024-05-25T06:24:46.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:24:46.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-05-25T06:24:46.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:45 smithi175 ceph-mon[84648]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-25T06:24:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:24:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:24:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:24:47.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:24:47.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:46 smithi175 ceph-mon[29436]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: Updating smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:24:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: Updating smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-05-25T06:24:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:24:47.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:46 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:24:48.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[29436]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:24:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:24:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-25T06:24:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[84648]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:24:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:24:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-25T06:24:48.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:47.725+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:48.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:24:48.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:48.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:48.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:48.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:48.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:48.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:48.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:48.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:24:48.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:48.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:24:48.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:48.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:48.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:47 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:47 smithi179 ceph-mon[67580]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:24:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:24:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:24:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-25T06:24:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:48 smithi175 ceph-mon[29436]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:24:49.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:49.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:48 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:49.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:49.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.078 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:48 smithi175 ceph-mon[29436]: Upgrade: It appears safe to stop mon.c 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:48 smithi175 ceph-mon[29436]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:48 smithi175 ceph-mon[84648]: Upgrade: It appears safe to stop mon.c 2024-05-25T06:24:49.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:48 smithi175 ceph-mon[84648]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:48 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:49.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:49.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:49.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:49.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:49.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:49.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:49.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:49.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:24:49.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:24:49.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:24:49.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:24:49.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:48 smithi179 ceph-mon[67580]: Upgrade: It appears safe to stop mon.c 2024-05-25T06:24:49.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:48 smithi179 ceph-mon[67580]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[29436]: Upgrade: Updating mon.c 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[29436]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[29436]: Deploying daemon mon.c on smithi175 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[84648]: Upgrade: Updating mon.c 2024-05-25T06:24:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[84648]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:50.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:24:50.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:24:50.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:50.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:50 smithi175 ceph-mon[84648]: Deploying daemon mon.c on smithi175 2024-05-25T06:24:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:50 smithi179 ceph-mon[67580]: Upgrade: Updating mon.c 2024-05-25T06:24:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:50 smithi179 ceph-mon[67580]: from='mgr.25663 ' entity='mgr.y' 2024-05-25T06:24:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:50 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:24:50.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:50 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:24:50.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:50 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:24:50.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:50 smithi179 ceph-mon[67580]: Deploying daemon mon.c on smithi175 2024-05-25T06:24:51.401 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:51 smithi175 ceph-mon[84648]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:51.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:51.402 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 systemd[1]: Stopping Ceph mon.c for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:24:51.402 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 ceph-mon[29436]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:51.402 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 ceph-mon[29436]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:51.776 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c[29432]: 2024-05-25T06:24:51.400+0000 7fefa81ab700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:24:51.776 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c[29432]: 2024-05-25T06:24:51.400+0000 7fefa81ab700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-05-25T06:24:51.776 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 podman[89513]: 2024-05-25 06:24:51.473334052 +0000 UTC m=+0.202982790 container died ea2c474b542abb2a09b0d99f754b245d1be5b31ea5789a818b08f0818d0c516b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, release=754, vcs-type=git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_CLEAN=True) 2024-05-25T06:24:51.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:51] ENGINE Bus STOPPING 2024-05-25T06:24:51.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:24:51.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:51] ENGINE Bus STOPPED 2024-05-25T06:24:51.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:51] ENGINE Bus STARTING 2024-05-25T06:24:51.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:51] ENGINE Serving on http://:::9283 2024-05-25T06:24:51.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:24:51] ENGINE Bus STARTED 2024-05-25T06:24:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:51 smithi179 ceph-mon[67580]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-25T06:24:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:24:52.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 podman[89513]: 2024-05-25 06:24:51.876207702 +0000 UTC m=+0.605856442 container cleanup ea2c474b542abb2a09b0d99f754b245d1be5b31ea5789a818b08f0818d0c516b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, architecture=x86_64, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, release=754) 2024-05-25T06:24:52.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:51 smithi175 bash[89513]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c 2024-05-25T06:24:52.475 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 podman[89536]: 2024-05-25 06:24:52.094354926 +0000 UTC m=+0.625379214 container remove ea2c474b542abb2a09b0d99f754b245d1be5b31ea5789a818b08f0818d0c516b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, name=centos-stream, version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-25T06:24:52.475 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.c.service: Deactivated successfully. 2024-05-25T06:24:52.475 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 systemd[1]: Stopped Ceph mon.c for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:24:52.476 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.c.service: Consumed 15.815s CPU time. 2024-05-25T06:24:52.603 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:24:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 systemd[1]: Starting Ceph mon.c for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:24:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 podman[89689]: 2024-05-25 06:24:52.703666203 +0000 UTC m=+0.113327591 container create 067b802471a7cd0e06cf656a37bb77b0b6140d4401b2e87362fe105a383970cb (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-25T06:24:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 podman[89689]: 2024-05-25 06:24:52.615032383 +0000 UTC m=+0.024693806 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:24:53.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 podman[89689]: 2024-05-25 06:24:52.956234122 +0000 UTC m=+0.365895535 container init 067b802471a7cd0e06cf656a37bb77b0b6140d4401b2e87362fe105a383970cb (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c) 2024-05-25T06:24:53.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 podman[89689]: 2024-05-25 06:24:52.960336491 +0000 UTC m=+0.369997895 container start 067b802471a7cd0e06cf656a37bb77b0b6140d4401b2e87362fe105a383970cb (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-05-25T06:24:53.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: set uid:gid to 167:167 (ceph:ceph) 2024-05-25T06:24:53.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc), process ceph-mon, pid 2 2024-05-25T06:24:53.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: pidfile_write: ignore empty --pid-file 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: load: jerasure load: lrc 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: RocksDB version: 7.9.2 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Git sha 0 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Compile date 2024-05-23 20:42:41 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: DB SUMMARY 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: DB Session ID: DYDX65AMUGE18OCJMELA 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: CURRENT file: CURRENT 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: IDENTITY file: IDENTITY 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3224 Bytes 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2495331 ; 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.error_if_exists: 0 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.create_if_missing: 0 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.paranoid_checks: 1 2024-05-25T06:24:53.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.env: 0x55803be60c60 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.fs: PosixFileSystem 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.info_log: 0x55803d201b20 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_file_opening_threads: 16 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.statistics: (nil) 2024-05-25T06:24:53.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.use_fsync: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_log_file_size: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.keep_log_file_num: 1000 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.recycle_log_file_num: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.allow_fallocate: 1 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.allow_mmap_reads: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.allow_mmap_writes: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.use_direct_reads: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.create_missing_column_families: 0 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.db_log_dir: 2024-05-25T06:24:53.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.wal_dir: 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.advise_random_on_open: 1 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.db_write_buffer_size: 0 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.write_buffer_manager: 0x55803d1d2820 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.rate_limiter: (nil) 2024-05-25T06:24:53.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.wal_recovery_mode: 2 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.enable_thread_tracking: 0 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.enable_pipelined_write: 0 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.unordered_write: 0 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.row_cache: None 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.wal_filter: None 2024-05-25T06:24:53.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (2m) 13s ago 25m 21.8M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (2m) 13s ago 24m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 13s ago 24m 61.9M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (4m) 13s ago 29m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (14m) 13s ago 31m 479M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (26s) 13s ago 31m 28.9M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (77s) 13s ago 29m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 starting - - - 2048M 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (3m) 13s ago 25m 7784k - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (3m) 13s ago 25m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (29m) 13s ago 29m 56.8M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:24:53.326 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (28m) 13s ago 28m 58.5M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (28m) 13s ago 28m 52.7M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (28m) 13s ago 28m 53.2M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (27m) 13s ago 27m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (27m) 13s ago 27m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (26m) 13s ago 26m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (26m) 13s ago 26m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (3m) 13s ago 25m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (24m) 13s ago 24m 77.1M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:24:53.327 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (24m) 13s ago 24m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.allow_ingest_behind: 0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.two_write_queues: 0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.manual_wal_flush: 0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.wal_compression: 0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.atomic_flush: 0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-25T06:24:53.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.log_readahead_size: 0 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.best_efforts_recovery: 0 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.allow_data_in_errors: 0 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.db_host_id: __hostname__ 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_background_jobs: 2 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_background_compactions: -1 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_subcompactions: 1 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-25T06:24:53.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_total_wal_size: 0 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_open_files: -1 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bytes_per_sync: 0 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_readahead_size: 0 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_background_flushes: -1 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Compression algorithms supported: 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-25T06:24:53.330 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kZSTD supported: 0 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kXpressCompression supported: 0 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kLZ4HCCompression supported: 1 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kZlibCompression supported: 1 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kSnappyCompression supported: 1 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kLZ4Compression supported: 1 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: kBZip2Compression supported: 0 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.merge_operator: 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_filter: None 2024-05-25T06:24:53.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_filter_factory: None 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.sst_partitioner_factory: None 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55803d201c40) 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: cache_index_and_filter_blocks: 1 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: pin_top_level_index_and_filter: 1 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: index_type: 0 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: data_block_index_type: 0 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: index_shortening: 1 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: checksum: 4 2024-05-25T06:24:53.332 INFO:journalctl@ceph.mon.c.smithi175.stdout: no_block_cache: 0 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_cache: 0x55803d203090 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_cache_name: BinnedLRUCache 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_cache_options: 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: capacity : 536870912 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: num_shard_bits : 4 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: strict_capacity_limit : 0 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: high_pri_pool_ratio: 0.000 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_cache_compressed: (nil) 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: persistent_cache: (nil) 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_size: 4096 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_size_deviation: 10 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_restart_interval: 16 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: index_block_restart_interval: 1 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: metadata_block_size: 4096 2024-05-25T06:24:53.333 INFO:journalctl@ceph.mon.c.smithi175.stdout: partition_filters: 0 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_delta_encoding: 1 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: filter_policy: bloomfilter 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: whole_key_filtering: 1 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: verify_compression: 0 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: read_amp_bytes_per_bit: 0 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: format_version: 5 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: enable_index_compression: 1 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: block_align: 0 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: max_auto_readahead_size: 262144 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: prepopulate_block_cache: 0 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: initial_auto_readahead_size: 8192 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout: num_file_reads_for_auto_readahead: 2 2024-05-25T06:24:53.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.write_buffer_size: 33554432 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_write_buffer_number: 2 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression: NoCompression 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression: Disabled 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.prefix_extractor: nullptr 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.num_levels: 7 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-25T06:24:53.335 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.level: 32767 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.strategy: 0 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.enabled: false 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-25T06:24:53.336 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.target_file_size_base: 67108864 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-25T06:24:53.337 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.arena_block_size: 1048576 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.disable_auto_compactions: 0 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.table_properties_collectors: 2024-05-25T06:24:53.338 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.inplace_update_support: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.bloom_locality: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.max_successive_merges: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.paranoid_file_checks: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.force_consistency_checks: 1 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.report_bg_io_stats: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.ttl: 2592000 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-25T06:24:53.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.enable_blob_files: false 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.min_blob_size: 0 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.blob_file_size: 268435456 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.blob_file_starting_level: 0 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32733, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5bcb6193-c284-4e57-9dae-9c748a727e67 2024-05-25T06:24:53.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618292991263, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618292997577, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1580914, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32739, "largest_seqno": 33181, "table_properties": {"data_size": 1577766, "index_size": 1558, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 5224, "raw_average_key_size": 24, "raw_value_size": 1572832, "raw_average_value_size": 7281, "num_data_blocks": 70, "num_entries": 216, "num_filter_entries": 216, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716618292, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5bcb6193-c284-4e57-9dae-9c748a727e67", "db_session_id": "DYDX65AMUGE18OCJMELA", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716618292997722, "job": 1, "event": "recovery_finished"} 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55803d2f2000 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: DB pointer 0x55803d2dc000 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout: ** DB Stats ** 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T06:24:53.341 INFO:journalctl@ceph.mon.c.smithi175.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: L0 1/0 1.51 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 510.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: L6 1/0 10.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: Sum 2/0 12.28 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 510.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 510.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: ** Compaction Stats [default] ** 2024-05-25T06:24:53.342 INFO:journalctl@ceph.mon.c.smithi175.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 510.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Cumulative compaction: 0.00 GB write, 170.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Interval compaction: 0.00 GB write, 170.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Block cache BinnedLRUCache@0x55803d203090#2 capacity: 512.00 MB usage: 2.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-05-25T06:24:53.343 INFO:journalctl@ceph.mon.c.smithi175.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:52 smithi175 ceph-mon[89733]: starting mon.c rank 1 at public addrs [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] at bind addrs [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???) e3 preinit fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???).mds e1 new map 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???).mds e1 print_map 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: e1 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: legacy client fscid: -1 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout: No filesystems configured 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???).osd e108 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-25T06:24:53.344 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???).osd e108 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:24:53.345 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???).osd e108 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:24:53.345 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???).osd e108 crush map has features 432629239337189376, adjusting msgr requires 2024-05-25T06:24:53.345 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 ceph-mon[89733]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-05-25T06:24:53.345 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 bash[89689]: 067b802471a7cd0e06cf656a37bb77b0b6140d4401b2e87362fe105a383970cb 2024-05-25T06:24:53.345 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:53 smithi175 systemd[1]: Started Ceph mon.c for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:24:53.724 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:24:53.725 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:24:53.726 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:24:53.726 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:24:53.726 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-05-25T06:24:53.726 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 4 2024-05-25T06:24:53.726 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:24:53.726 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:24:53.727 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:24:53] "GET /metrics HTTP/1.1" 200 38017 "" "Prometheus/2.43.0" 2024-05-25T06:24:58.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:24:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:24:58.005+0000 7f7edf5ab700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-05-25T06:25:00.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: mon.c calling monitor election 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: mon.b calling monitor election 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: mon.c calling monitor election 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: monmap epoch 4 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: last_changed 2024-05-25T06:24:53.728140+0000 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: min_mon_release 19 (squid) 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: election_strategy: 1 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: fsmap 2024-05-25T06:25:00.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: mgrmap e45: y(active, since 23s), standbys: x 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: mon.a calling monitor election 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: monmap epoch 4 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: last_changed 2024-05-25T06:24:53.728140+0000 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: min_mon_release 19 (squid) 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: election_strategy: 1 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:25:00.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: fsmap 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: mgrmap e45: y(active, since 23s), standbys: x 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:00.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:24:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: mon.c calling monitor election 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: mon.b calling monitor election 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: mon.c calling monitor election 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: monmap epoch 4 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: last_changed 2024-05-25T06:24:53.728140+0000 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: min_mon_release 19 (squid) 2024-05-25T06:25:00.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: election_strategy: 1 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: fsmap 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: mgrmap e45: y(active, since 23s), standbys: x 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: mon.a calling monitor election 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: monmap epoch 4 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: last_changed 2024-05-25T06:24:53.728140+0000 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:25:00.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: min_mon_release 19 (squid) 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: election_strategy: 1 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: fsmap 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: mgrmap e45: y(active, since 23s), standbys: x 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: mon.c calling monitor election 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: mon.b calling monitor election 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: mon.c calling monitor election 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: monmap epoch 4 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: last_changed 2024-05-25T06:24:53.728140+0000 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: min_mon_release 19 (squid) 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: election_strategy: 1 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: fsmap 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: mgrmap e45: y(active, since 23s), standbys: x 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:25:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: mon.a calling monitor election 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: monmap epoch 4 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: last_changed 2024-05-25T06:24:53.728140+0000 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: created 2024-05-25T05:53:32.512282+0000 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: min_mon_release 19 (squid) 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: election_strategy: 1 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: 0: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.a 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: 1: [v2:172.21.15.175:3301/0,v1:172.21.15.175:6790/0] mon.c 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: 2: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: fsmap 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: osdmap e108: 8 total, 8 up, 8 in 2024-05-25T06:25:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: mgrmap e45: y(active, since 23s), standbys: x 2024-05-25T06:25:00.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:25:00.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:25:00.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:25:00.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:00.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:00.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:24:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:00.075 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:25:00.075 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:25:00.075 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:25:00.075 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi175", 2024-05-25T06:25:00.075 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:25:00.075 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "1/2 daemons upgraded", 2024-05-25T06:25:00.076 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading mon daemons", 2024-05-25T06:25:00.076 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:25:00.076 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:25:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:00 smithi179 ceph-mon[67580]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:01.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:00 smithi179 ceph-mon[67580]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:01.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:00 smithi175 ceph-mon[84648]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:01.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:00 smithi175 ceph-mon[84648]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:00 smithi175 ceph-mon[89733]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:00 smithi175 ceph-mon[89733]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:03.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:02 smithi179 ceph-mon[67580]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:03.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:02 smithi175 ceph-mon[84648]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:03.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:02 smithi175 ceph-mon[89733]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:25:03] "GET /metrics HTTP/1.1" 200 38017 "" "Prometheus/2.43.0" 2024-05-25T06:25:04.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:04 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:04.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:04 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:04.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:04 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:04.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:04 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:04.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:04 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:04.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:04 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:05 smithi179 ceph-mon[67580]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:05.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:05 smithi175 ceph-mon[89733]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:05.581 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:05.581 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:05.582 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:05 smithi175 ceph-mon[84648]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:05.582 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:05.582 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:06.461 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:06.462 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:07.287 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:07 smithi175 ceph-mon[89733]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:07.287 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:07 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:07.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:07 smithi175 ceph-mon[84648]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:07.287 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:07 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:07 smithi179 ceph-mon[67580]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:07 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:08.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:08.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:08.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:08 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:25:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:08.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:08 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:09.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[84648]: Reconfiguring mon.a (monmap changed)... 2024-05-25T06:25:09.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[84648]: Reconfiguring daemon mon.a on smithi175 2024-05-25T06:25:09.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:09.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:09.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:25:09.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:25:09.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:09.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[89733]: Reconfiguring mon.a (monmap changed)... 2024-05-25T06:25:09.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[89733]: Reconfiguring daemon mon.a on smithi175 2024-05-25T06:25:09.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:09.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:09.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:25:09.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:25:09.738 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:09 smithi179 ceph-mon[67580]: Reconfiguring mon.a (monmap changed)... 2024-05-25T06:25:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:09 smithi179 ceph-mon[67580]: Reconfiguring daemon mon.a on smithi175 2024-05-25T06:25:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:25:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:25:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:10.736 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:10 smithi175 ceph-mon[89733]: Reconfiguring mgr.y (monmap changed)... 2024-05-25T06:25:10.736 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:10 smithi175 ceph-mon[89733]: Reconfiguring daemon mgr.y on smithi175 2024-05-25T06:25:10.737 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:10 smithi175 ceph-mon[89733]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:10.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:10 smithi175 ceph-mon[84648]: Reconfiguring mgr.y (monmap changed)... 2024-05-25T06:25:10.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:10 smithi175 ceph-mon[84648]: Reconfiguring daemon mgr.y on smithi175 2024-05-25T06:25:10.737 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:10 smithi175 ceph-mon[84648]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:10 smithi179 ceph-mon[67580]: Reconfiguring mgr.y (monmap changed)... 2024-05-25T06:25:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:10 smithi179 ceph-mon[67580]: Reconfiguring daemon mgr.y on smithi175 2024-05-25T06:25:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:10 smithi179 ceph-mon[67580]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: Reconfiguring mon.c (monmap changed)... 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: Reconfiguring daemon mon.c on smithi175 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[89733]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:12.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: Reconfiguring mon.c (monmap changed)... 2024-05-25T06:25:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: Reconfiguring daemon mon.c on smithi175 2024-05-25T06:25:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:11 smithi175 ceph-mon[84648]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-05-25T06:25:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: Reconfiguring mon.c (monmap changed)... 2024-05-25T06:25:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:12.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: Reconfiguring daemon mon.c on smithi175 2024-05-25T06:25:12.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:11 smithi179 ceph-mon[67580]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-05-25T06:25:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:13.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:13 smithi179 ceph-mon[67580]: Reconfiguring osd.0 (monmap changed)... 2024-05-25T06:25:13.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T06:25:13.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:13.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:13 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.0 on smithi175 2024-05-25T06:25:13.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:25:13] "GET /metrics HTTP/1.1" 200 38024 "" "Prometheus/2.43.0" 2024-05-25T06:25:13.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[89733]: Reconfiguring osd.0 (monmap changed)... 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.0 on smithi175 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[84648]: Reconfiguring osd.0 (monmap changed)... 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:13 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.0 on smithi175 2024-05-25T06:25:14.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:14 smithi179 ceph-mon[67580]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:14.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:14.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:14.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T06:25:14.687 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[89733]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[84648]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:14.741 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:14.742 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T06:25:14.742 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:15 smithi179 ceph-mon[67580]: Reconfiguring osd.1 (monmap changed)... 2024-05-25T06:25:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:15 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.1 on smithi175 2024-05-25T06:25:15.802 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:15 smithi175 ceph-mon[89733]: Reconfiguring osd.1 (monmap changed)... 2024-05-25T06:25:15.802 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:15 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.1 on smithi175 2024-05-25T06:25:15.802 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:15 smithi175 ceph-mon[84648]: Reconfiguring osd.1 (monmap changed)... 2024-05-25T06:25:15.803 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:15 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.1 on smithi175 2024-05-25T06:25:16.738 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[84648]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:16.738 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:16.738 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:16.738 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T06:25:16.739 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:16.739 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[89733]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:16.739 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:16.739 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:16.739 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T06:25:16.739 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:16.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:16 smithi179 ceph-mon[67580]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:16.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:16.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:16.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T06:25:16.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:17.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:17 smithi179 ceph-mon[67580]: Reconfiguring osd.2 (monmap changed)... 2024-05-25T06:25:17.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:17 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.2 on smithi175 2024-05-25T06:25:17.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:17 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:17.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:17 smithi175 ceph-mon[89733]: Reconfiguring osd.2 (monmap changed)... 2024-05-25T06:25:17.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:17 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.2 on smithi175 2024-05-25T06:25:17.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:17 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:17 smithi175 ceph-mon[84648]: Reconfiguring osd.2 (monmap changed)... 2024-05-25T06:25:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:17 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.2 on smithi175 2024-05-25T06:25:17.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:17 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:18.647 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[84648]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:25:18.647 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:18.647 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:18.647 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T06:25:18.647 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:18.648 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[89733]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:25:18.648 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:18.648 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:18.648 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T06:25:18.648 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:18 smithi179 ceph-mon[67580]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:25:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T06:25:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:19.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[89733]: Reconfiguring osd.3 (monmap changed)... 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.3 on smithi175 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[84648]: Reconfiguring osd.3 (monmap changed)... 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.3 on smithi175 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:19.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:25:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:19 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:19 smithi179 ceph-mon[67580]: Reconfiguring osd.3 (monmap changed)... 2024-05-25T06:25:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:19 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.3 on smithi175 2024-05-25T06:25:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:19 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:19 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:19 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:25:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:19 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:20.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:20 smithi175 ceph-mon[89733]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:20 smithi175 ceph-mon[89733]: Reconfiguring rgw.foo.smithi175.iwhmky (monmap changed)... 2024-05-25T06:25:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:20 smithi175 ceph-mon[89733]: Reconfiguring daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:25:20.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:20 smithi175 ceph-mon[84648]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:20.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:20 smithi175 ceph-mon[84648]: Reconfiguring rgw.foo.smithi175.iwhmky (monmap changed)... 2024-05-25T06:25:20.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:20 smithi175 ceph-mon[84648]: Reconfiguring daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:25:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:20 smithi179 ceph-mon[67580]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:20 smithi179 ceph-mon[67580]: Reconfiguring rgw.foo.smithi175.iwhmky (monmap changed)... 2024-05-25T06:25:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:20 smithi179 ceph-mon[67580]: Reconfiguring daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:25:21.747 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:21.747 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:21.747 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:21.747 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:21.747 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:21.747 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:21.747 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:21.748 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:21.748 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:21.748 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:22.775 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:22 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:22.776 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:22 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:22.776 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:22 smithi175 ceph-mon[89733]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:22.776 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:22 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:22.776 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:22 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:22.776 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:22 smithi175 ceph-mon[84648]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:22 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:22 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:22 smithi179 ceph-mon[67580]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:23.056 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:25:22.913+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:23.056 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:25:23.056 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:23.056 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:23.057 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:23.058 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:23.059 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:23.060 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:23.061 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:23.061 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:23.061 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:23.061 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:23.061 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:22 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:23.770 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:23 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:23.770 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:23 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:23.771 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:23 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:25:23] "GET /metrics HTTP/1.1" 200 38023 "" "Prometheus/2.43.0" 2024-05-25T06:25:23.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:23 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:23 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:23 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:23 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-25T06:25:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:23 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-25T06:25:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:23 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:24 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:24.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:24.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:24.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:24.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:24.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:24.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:24.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:24.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:24.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:24.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:24.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:24.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:24.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:24.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:24.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:24.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:24.792 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:24.793 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.793 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:24 smithi179 ceph-mon[67580]: Reconfiguring mon.b (monmap changed)... 2024-05-25T06:25:24.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:24 smithi179 ceph-mon[67580]: Reconfiguring daemon mon.b on smithi179 2024-05-25T06:25:24.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:24 smithi179 ceph-mon[67580]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:24.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:24 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:24.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:24 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:24.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:24.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:24.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:24.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:24.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:24.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:24.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[84648]: Reconfiguring mon.b (monmap changed)... 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[84648]: Reconfiguring daemon mon.b on smithi179 2024-05-25T06:25:24.828 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[84648]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[89733]: Reconfiguring mon.b (monmap changed)... 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[89733]: Reconfiguring daemon mon.b on smithi179 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[89733]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:24.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:24 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:25.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:25 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:25:25.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:25 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:25:25.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:25 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:25.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:25 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:25:25.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:25 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:25:25.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:25 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:25 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-25T06:25:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:25 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-25T06:25:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:25 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:26.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:26 smithi179 ceph-mon[67580]: Reconfiguring mgr.x (monmap changed)... 2024-05-25T06:25:26.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:26 smithi179 ceph-mon[67580]: Reconfiguring daemon mgr.x on smithi179 2024-05-25T06:25:26.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:26 smithi179 ceph-mon[67580]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:26.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:26.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:26.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T06:25:26.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:26.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[84648]: Reconfiguring mgr.x (monmap changed)... 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[84648]: Reconfiguring daemon mgr.x on smithi179 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[84648]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[89733]: Reconfiguring mgr.x (monmap changed)... 2024-05-25T06:25:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[89733]: Reconfiguring daemon mgr.x on smithi179 2024-05-25T06:25:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[89733]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T06:25:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:27.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:27 smithi179 ceph-mon[67580]: Reconfiguring osd.4 (monmap changed)... 2024-05-25T06:25:27.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:27 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.4 on smithi179 2024-05-25T06:25:27.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:27 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:27.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:27 smithi175 ceph-mon[89733]: Reconfiguring osd.4 (monmap changed)... 2024-05-25T06:25:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:27 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.4 on smithi179 2024-05-25T06:25:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:27 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:27 smithi175 ceph-mon[84648]: Reconfiguring osd.4 (monmap changed)... 2024-05-25T06:25:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:27 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.4 on smithi179 2024-05-25T06:25:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:27 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:28.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:28 smithi179 ceph-mon[67580]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:28.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:28 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:28.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:28 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:28.532 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:28 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T06:25:28.532 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:28 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[84648]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T06:25:28.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[89733]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:28.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:28.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:28.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T06:25:28.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:28 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:29 smithi179 ceph-mon[67580]: Reconfiguring osd.5 (monmap changed)... 2024-05-25T06:25:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:29 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.5 on smithi179 2024-05-25T06:25:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:29 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:29 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:29.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:29 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T06:25:29.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:29 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[89733]: Reconfiguring osd.5 (monmap changed)... 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.5 on smithi179 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[84648]: Reconfiguring osd.5 (monmap changed)... 2024-05-25T06:25:29.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.5 on smithi179 2024-05-25T06:25:29.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:29.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:29.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T06:25:29.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:29 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:30.440 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:25:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:30 smithi179 ceph-mon[67580]: Reconfiguring osd.6 (monmap changed)... 2024-05-25T06:25:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:30 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.6 on smithi179 2024-05-25T06:25:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:30 smithi179 ceph-mon[67580]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:30.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:30 smithi175 ceph-mon[89733]: Reconfiguring osd.6 (monmap changed)... 2024-05-25T06:25:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:30 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.6 on smithi179 2024-05-25T06:25:30.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:30 smithi175 ceph-mon[89733]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:30 smithi175 ceph-mon[84648]: Reconfiguring osd.6 (monmap changed)... 2024-05-25T06:25:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:30 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.6 on smithi179 2024-05-25T06:25:30.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:30 smithi175 ceph-mon[84648]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:31.096 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:25:31.096 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (3m) 28s ago 25m 21.8M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:25:31.096 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (3m) 51s ago 25m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:25:31.096 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 28s ago 24m 62.5M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:25:31.096 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (4m) 51s ago 30m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (14m) 28s ago 31m 498M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (64s) 28s ago 31m 40.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (114s) 51s ago 30m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (38s) 28s ago 30m 23.3M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (4m) 28s ago 26m 14.4M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (4m) 51s ago 26m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (29m) 28s ago 29m 57.4M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (29m) 28s ago 29m 58.9M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (29m) 28s ago 29m 53.1M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (28m) 28s ago 28m 53.2M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (28m) 51s ago 28m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (27m) 51s ago 27m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (27m) 51s ago 27m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:25:31.097 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (27m) 51s ago 27m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:25:31.098 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (3m) 51s ago 26m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:25:31.098 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (24m) 28s ago 24m 77.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:25:31.098 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (24m) 51s ago 24m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:25:31.500 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:25:31.500 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:25:31.501 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-25T06:25:31.502 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 5 2024-05-25T06:25:31.502 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:25:31.502 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:25:31.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:31.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:31.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: Reconfiguring osd.7 (monmap changed)... 2024-05-25T06:25:31.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T06:25:31.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:31.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: Reconfiguring daemon osd.7 on smithi179 2024-05-25T06:25:31.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:31 smithi179 ceph-mon[67580]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:31.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: Reconfiguring osd.7 (monmap changed)... 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: Reconfiguring daemon osd.7 on smithi179 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[89733]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:31.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: Reconfiguring osd.7 (monmap changed)... 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: Reconfiguring daemon osd.7 on smithi179 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:31.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:31 smithi175 ceph-mon[84648]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi175", 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [ 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "mon" 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "2/2 daemons upgraded", 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading mon daemons", 2024-05-25T06:25:31.839 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:25:31.840 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:25:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1202483093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: Reconfiguring rgw.foo.smithi179.hopgfy (monmap changed)... 2024-05-25T06:25:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:25:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:32.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:32 smithi179 ceph-mon[67580]: Reconfiguring daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:25:32.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1202483093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: Reconfiguring rgw.foo.smithi179.hopgfy (monmap changed)... 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[84648]: Reconfiguring daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1202483093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:32.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:32.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: Reconfiguring rgw.foo.smithi179.hopgfy (monmap changed)... 2024-05-25T06:25:32.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:25:32.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:32.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:32 smithi175 ceph-mon[89733]: Reconfiguring daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:25:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:33 smithi179 ceph-mon[67580]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:33 smithi175 ceph-mon[84648]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:33.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:25:33] "GET /metrics HTTP/1.1" 200 38023 "" "Prometheus/2.43.0" 2024-05-25T06:25:33.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:33 smithi175 ceph-mon[89733]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mon 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all crash 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mds 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nfs 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all node-exporter 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all prometheus 2024-05-25T06:25:35.288 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all alertmanager 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all grafana 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all loki 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all promtail 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Finalizing container_image settings 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:25:35.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.291 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: Upgrade: Complete! 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:35.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mon 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all crash 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mds 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nfs 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mon 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all crash 2024-05-25T06:25:35.323 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mds 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:25:35.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nfs 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all node-exporter 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.325 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all prometheus 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all alertmanager 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all grafana 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.326 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all loki 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all promtail 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Finalizing container_image settings 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.327 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all node-exporter 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all prometheus 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all alertmanager 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all grafana 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all loki 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all promtail 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Finalizing container_image settings 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:25:35.329 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:25:35.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.331 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:25:35.332 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: Upgrade: Complete! 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:25:35.333 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: Upgrade: Complete! 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:25:35.334 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:35.335 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:35 smithi179 ceph-mon[67580]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-05-25T06:25:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:35 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:35 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:36.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[84648]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-05-25T06:25:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:36.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[89733]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-05-25T06:25:36.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:36.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:36.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:36.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:36.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:25:36.565+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:36.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:25:36.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:36.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:36.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:36.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:36.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:36.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:36.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:36.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:36.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:36.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:25:36.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:36.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:37.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:37.311 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:25:37.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:25:38.265 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:38.265 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:38.266 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:38.267 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:38.268 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.269 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.270 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:38.271 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.272 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.273 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.274 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[84648]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[84648]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[89733]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[89733]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:38.275 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:37 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:37 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:37 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:38.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:38.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:37 smithi179 ceph-mon[67580]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:37 smithi179 ceph-mon[67580]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:37 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:25:38.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:37 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:25:38.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:25:38.378+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:25:38.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:38.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:38.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:38.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:38.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:38 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:38 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:39.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:39.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:25:39.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:38 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:25:39.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:39.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:38 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:39.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:39.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:25:39.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:25:39.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:25:40.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:39 smithi179 ceph-mon[67580]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:40.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:39 smithi175 ceph-mon[84648]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:40.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:39 smithi175 ceph-mon[89733]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:42.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:41 smithi179 ceph-mon[67580]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:42.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:41 smithi175 ceph-mon[84648]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:42.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:41 smithi175 ceph-mon[89733]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:25:43] "GET /metrics HTTP/1.1" 200 38022 "" "Prometheus/2.43.0" 2024-05-25T06:25:44.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:43 smithi179 ceph-mon[67580]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:44.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:43 smithi175 ceph-mon[89733]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:43 smithi175 ceph-mon[84648]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:25:46.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:46 smithi179 ceph-mon[67580]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:25:46.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:46 smithi175 ceph-mon[89733]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:25:46.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:46 smithi175 ceph-mon[84648]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:25:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:48 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:48 smithi179 ceph-mon[67580]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:25:48.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:48 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:48.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:48 smithi175 ceph-mon[89733]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:25:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:48 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:48.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:48 smithi175 ceph-mon[84648]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:25:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:50 smithi179 ceph-mon[67580]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:50.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:50 smithi175 ceph-mon[89733]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:50 smithi175 ceph-mon[84648]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:51.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:51.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:51.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:25:52.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:52 smithi179 ceph-mon[67580]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:52.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:52 smithi175 ceph-mon[84648]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:52.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:52 smithi175 ceph-mon[89733]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:53.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:25:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:25:53] "GET /metrics HTTP/1.1" 200 37954 "" "Prometheus/2.43.0" 2024-05-25T06:25:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:54 smithi179 ceph-mon[67580]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:54.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:54 smithi175 ceph-mon[89733]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:54 smithi175 ceph-mon[84648]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:56.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:56 smithi179 ceph-mon[67580]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:56.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:56 smithi175 ceph-mon[84648]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:56.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:56 smithi175 ceph-mon[89733]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:25:58.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:58 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:58.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:25:58 smithi179 ceph-mon[67580]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:58.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:58 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:58.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:25:58 smithi175 ceph-mon[84648]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:25:58.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:58 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:25:58.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:25:58 smithi175 ceph-mon[89733]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:00 smithi179 ceph-mon[67580]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:00.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:00 smithi175 ceph-mon[84648]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:00.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:00 smithi175 ceph-mon[89733]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:02.204 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:02 smithi175 ceph-mon[89733]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:02.204 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:02 smithi175 ceph-mon[84648]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:02 smithi179 ceph-mon[67580]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:02.859 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:26:03.171 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:03 smithi175 ceph-mon[89733]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:03.171 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:03 smithi175 ceph-mon[84648]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:03.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:03 smithi179 ceph-mon[67580]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:26:03] "GET /metrics HTTP/1.1" 200 37954 "" "Prometheus/2.43.0" 2024-05-25T06:26:04.241 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:04 smithi175 ceph-mon[84648]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:04.242 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:04 smithi175 ceph-mon[89733]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:04.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:04 smithi179 ceph-mon[67580]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:04.979 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:26:04.979 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (4m) 61s ago 26m 21.8M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (3m) 84s ago 25m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 61s ago 25m 62.5M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (5m) 84s ago 30m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (15m) 61s ago 32m 498M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (98s) 61s ago 32m 40.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (2m) 84s ago 31m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (72s) 61s ago 31m 23.3M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (5m) 61s ago 26m 14.4M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (4m) 84s ago 26m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (30m) 61s ago 30m 57.4M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (30m) 61s ago 30m 58.9M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:26:04.980 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (29m) 61s ago 29m 53.1M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (29m) 61s ago 29m 53.2M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (28m) 84s ago 28m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (28m) 84s ago 28m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (28m) 84s ago 28m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (27m) 84s ago 27m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (4m) 84s ago 27m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (25m) 61s ago 25m 77.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:26:04.981 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (25m) 84s ago 25m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:26:05.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-05-25T06:26:06.186 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:06 smithi175 ceph-mon[84648]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:06.186 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:06 smithi175 ceph-mon[84648]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:06.186 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:06.186 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:06 smithi175 ceph-mon[89733]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:06.186 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:06 smithi175 ceph-mon[89733]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:06.186 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:06 smithi179 ceph-mon[67580]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:06 smithi179 ceph-mon[67580]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:06.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:07.652 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:26:08.223 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:08 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:08.223 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:08 smithi175 ceph-mon[84648]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:08.223 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:08 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1320289637' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:08.224 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:08 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:08.224 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:08 smithi175 ceph-mon[89733]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:08.224 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:08 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1320289637' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:08.248 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-05-25T06:26:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:08 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:08 smithi179 ceph-mon[67580]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:08.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:08 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1320289637' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:10.227 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)" 2024-05-25T06:26:10.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:10 smithi175 ceph-mon[84648]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:10.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:10 smithi175 ceph-mon[89733]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:10.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:10 smithi179 ceph-mon[67580]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:10.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-05-25T06:26:11.195 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:11 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/918769256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:11.196 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:11 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/918769256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:11 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/918769256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:12.104 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:12 smithi175 ceph-mon[84648]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:12.104 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:12 smithi175 ceph-mon[89733]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:12 smithi179 ceph-mon[67580]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:13.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:26:13] "GET /metrics HTTP/1.1" 200 38022 "" "Prometheus/2.43.0" 2024-05-25T06:26:14.140 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:14 smithi175 ceph-mon[89733]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:14.140 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:14 smithi175 ceph-mon[89733]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:14.140 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:14 smithi175 ceph-mon[84648]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:14.140 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:14 smithi175 ceph-mon[84648]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:14.337 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:14 smithi179 ceph-mon[67580]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:14.337 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:14 smithi179 ceph-mon[67580]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:14.903 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:26:15.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:26:16.179 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:16 smithi175 ceph-mon[89733]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:16.179 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:16 smithi175 ceph-mon[84648]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:16 smithi179 ceph-mon[67580]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:17.635 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:26:17.635 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": null, 2024-05-25T06:26:17.635 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": false, 2024-05-25T06:26:17.635 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "", 2024-05-25T06:26:17.635 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:26:17.636 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": null, 2024-05-25T06:26:17.636 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "", 2024-05-25T06:26:17.636 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:26:17.636 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:26:18.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:26:18.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:18 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:18.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:18 smithi175 ceph-mon[89733]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:18.312 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:18 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:18.313 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:18 smithi175 ceph-mon[84648]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:18 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:18 smithi179 ceph-mon[67580]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:19.312 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:19 smithi175 ceph-mon[89733]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:19.313 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:19 smithi175 ceph-mon[84648]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:19 smithi179 ceph-mon[67580]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:20.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:20 smithi175 ceph-mon[84648]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:20.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:20 smithi175 ceph-mon[89733]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:20.430 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:26:20.430 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:26:20.430 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:26:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:20 smithi179 ceph-mon[67580]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:21.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-05-25T06:26:21.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:21 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1226075904' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:26:21.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:21 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1226075904' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:26:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:21 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1226075904' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:26:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:22.377 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:22 smithi175 ceph-mon[89733]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:22.436 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:22 smithi175 ceph-mon[84648]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:22.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:22 smithi179 ceph-mon[67580]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:26:23] "GET /metrics HTTP/1.1" 200 38019 "" "Prometheus/2.43.0" 2024-05-25T06:26:24.161 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:24 smithi175 ceph-mon[84648]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:24.161 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:24 smithi175 ceph-mon[84648]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:24.161 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:24 smithi175 ceph-mon[89733]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:24.161 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:24 smithi175 ceph-mon[89733]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:24.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:24 smithi179 ceph-mon[67580]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:24.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:24 smithi179 ceph-mon[67580]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:25.205 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:26:25.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-25T06:26:26.448 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[89733]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[89733]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[84648]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[84648]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.449 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:26 smithi179 ceph-mon[67580]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:26:26.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:26 smithi179 ceph-mon[67580]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:26:26.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:26.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:26.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:27.443 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:27.443 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:26:27.443 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-05-25T06:26:27.443 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 0 op/s 2024-05-25T06:26:27.443 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:27.443 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:26:27.443 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 0 op/s 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:27.444 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:27 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:26:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:26:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-05-25T06:26:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 0 op/s 2024-05-25T06:26:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:26:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:26:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:27.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:27 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:26:27.942 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:26:28.481 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:28 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:28.482 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:28 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:28.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:28 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:28.606 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (4m) 85s ago 26m 21.8M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (4m) 108s ago 26m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 85s ago 25m 62.5M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (5m) 108s ago 31m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (15m) 85s ago 32m 498M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (2m) 85s ago 32m 40.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (2m) 108s ago 31m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (95s) 85s ago 31m 23.3M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (5m) 85s ago 27m 14.4M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (5m) 108s ago 27m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (30m) 85s ago 30m 57.4M 2992M 17.2.0 e1d6a67b021e f3995535e1d4 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (30m) 85s ago 30m 58.9M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:26:28.607 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (30m) 85s ago 30m 53.1M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (29m) 85s ago 29m 53.2M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (29m) 108s ago 29m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (28m) 108s ago 28m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (28m) 108s ago 28m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (28m) 108s ago 28m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (4m) 108s ago 27m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (25m) 85s ago 25m 77.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:26:28.608 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (25m) 108s ago 25m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:26:29.033 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:26:29.034 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:26:29.034 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:26:29.034 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:26:29.034 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-25T06:26:29.034 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 5 2024-05-25T06:26:29.034 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:26:29.034 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:26:29.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:26:28.957+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:29.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:26:29.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:29.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:29.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:29.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:29.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:29.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:29.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:29.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:29.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:28 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:29.383 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:26:29.383 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", 2024-05-25T06:26:29.383 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:26:29.383 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-05-25T06:26:29.383 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:26:29.383 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "", 2024-05-25T06:26:29.384 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 image", 2024-05-25T06:26:29.384 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:26:29.384 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:26:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:29 smithi179 ceph-mon[67580]: from='client.44244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:29 smithi179 ceph-mon[67580]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:29 smithi179 ceph-mon[67580]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:29 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3076632806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[84648]: from='client.44244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[84648]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[84648]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3076632806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[89733]: from='client.44244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[89733]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[89733]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:29.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:29 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3076632806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:30 smithi179 ceph-mon[67580]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:30 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:30.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:30.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:30.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:26:30.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:30.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:30.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:30.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:30.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:30.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:30.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:30 smithi179 ceph-mon[67580]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:26:30.541 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:30 smithi179 ceph-mon[67580]: from='client.44268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[84648]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:30.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:30.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:30.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:30.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[89733]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:30.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:30.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:30.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:30.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[84648]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[84648]: from='client.44268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:30.580 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:30.581 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:30.581 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:30.581 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:30.581 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:30.581 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[89733]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:26:30.581 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:30 smithi175 ceph-mon[89733]: from='client.44268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-25T06:26:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:31.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-25T06:26:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:32 smithi179 ceph-mon[67580]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:26:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:32 smithi179 ceph-mon[67580]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:26:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:32 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mgr 2024-05-25T06:26:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:32 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mon 2024-05-25T06:26:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:32 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all crash 2024-05-25T06:26:32.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:32 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-25T06:26:32.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:32 smithi179 ceph-mon[67580]: Upgrade: osd.0 is safe to restart 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[84648]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[84648]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mgr 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mon 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all crash 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[84648]: Upgrade: osd.0 is safe to restart 2024-05-25T06:26:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[89733]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:26:32.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[89733]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:26:32.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mgr 2024-05-25T06:26:32.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mon 2024-05-25T06:26:32.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all crash 2024-05-25T06:26:32.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-25T06:26:32.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:32 smithi175 ceph-mon[89733]: Upgrade: osd.0 is safe to restart 2024-05-25T06:26:33.494 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[89733]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:33.494 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:33.494 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T06:26:33.494 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:33.495 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[84648]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:33.495 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:33.495 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T06:26:33.495 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:33 smithi179 ceph-mon[67580]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-25T06:26:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:26:33] "GET /metrics HTTP/1.1" 200 38019 "" "Prometheus/2.43.0" 2024-05-25T06:26:34.458 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:34 smithi175 ceph-mon[84648]: Upgrade: Updating osd.0 2024-05-25T06:26:34.458 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:34 smithi175 ceph-mon[84648]: Deploying daemon osd.0 on smithi175 2024-05-25T06:26:34.459 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:34 smithi175 ceph-mon[89733]: Upgrade: Updating osd.0 2024-05-25T06:26:34.459 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:34 smithi175 ceph-mon[89733]: Deploying daemon osd.0 on smithi175 2024-05-25T06:26:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:34 smithi179 ceph-mon[67580]: Upgrade: Updating osd.0 2024-05-25T06:26:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:34 smithi179 ceph-mon[67580]: Deploying daemon osd.0 on smithi175 2024-05-25T06:26:35.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:35 smithi179 ceph-mon[67580]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:35.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:35 smithi175 ceph-mon[84648]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:35.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:35 smithi175 ceph-mon[89733]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:26:36.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:36.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:36.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:36 smithi175 systemd[1]: Stopping Ceph osd.0 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:26:36.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[32276]: 2024-05-25T06:26:36.252+0000 7fcb1891a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:26:36.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[32276]: 2024-05-25T06:26:36.252+0000 7fcb1891a700 -1 osd.0 108 *** Got signal Terminated *** 2024-05-25T06:26:36.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[32276]: 2024-05-25T06:26:36.252+0000 7fcb1891a700 -1 osd.0 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:26:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:37.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:37 smithi179 ceph-mon[67580]: osd.0 marked itself down and dead 2024-05-25T06:26:37.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:37 smithi179 ceph-mon[67580]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:26:37.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:37 smithi175 ceph-mon[84648]: osd.0 marked itself down and dead 2024-05-25T06:26:37.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:37 smithi175 ceph-mon[84648]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:26:37.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:37 smithi175 ceph-mon[89733]: osd.0 marked itself down and dead 2024-05-25T06:26:37.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:37 smithi175 ceph-mon[89733]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:26:37.847 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:37 smithi175 podman[97630]: 2024-05-25 06:26:37.597234546 +0000 UTC m=+1.431326970 container died f3995535e1d44f4a4f48a00afa28b831af15208d37a66b2be687f555a3b3c563 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, maintainer=Guillaume Abrioux , release=754, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc.) 2024-05-25T06:26:38.214 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:37 smithi175 podman[97630]: 2024-05-25 06:26:37.946884426 +0000 UTC m=+1.780976840 container cleanup f3995535e1d44f4a4f48a00afa28b831af15208d37a66b2be687f555a3b3c563 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, maintainer=Guillaume Abrioux , architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vendor=Red Hat, Inc., distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.component=centos-stream-container, version=8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8) 2024-05-25T06:26:38.214 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:37 smithi175 bash[97630]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0 2024-05-25T06:26:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:38 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:38 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:26:38.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:38 smithi179 ceph-mon[67580]: osdmap e109: 8 total, 7 up, 8 in 2024-05-25T06:26:38.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:38 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:38 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:26:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:38 smithi175 ceph-mon[84648]: osdmap e109: 8 total, 7 up, 8 in 2024-05-25T06:26:38.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:38 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:38.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:38 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:26:38.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:38 smithi175 ceph-mon[89733]: osdmap e109: 8 total, 7 up, 8 in 2024-05-25T06:26:38.570 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:38 smithi175 podman[97658]: 2024-05-25 06:26:38.214573019 +0000 UTC m=+0.618069164 container remove f3995535e1d44f4a4f48a00afa28b831af15208d37a66b2be687f555a3b3c563 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, ceph=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, io.openshift.expose-services=) 2024-05-25T06:26:38.943 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:38 smithi175 podman[97747]: 2024-05-25 06:26:38.480045605 +0000 UTC m=+0.017192352 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:26:38.943 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:38 smithi175 podman[97747]: 2024-05-25 06:26:38.599685864 +0000 UTC m=+0.136832600 container create 0442176c73aac2f6f893aa94babe2c9b6df4e7af3893af496d45597d1e546a58 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-25T06:26:38.943 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:38 smithi175 podman[97747]: 2024-05-25 06:26:38.818765813 +0000 UTC m=+0.355912554 container init 0442176c73aac2f6f893aa94babe2c9b6df4e7af3893af496d45597d1e546a58 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-05-25T06:26:38.943 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:38 smithi175 podman[97747]: 2024-05-25 06:26:38.823565158 +0000 UTC m=+0.360711894 container start 0442176c73aac2f6f893aa94babe2c9b6df4e7af3893af496d45597d1e546a58 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-25T06:26:38.943 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:38 smithi175 podman[97747]: 2024-05-25 06:26:38.943209168 +0000 UTC m=+0.480355907 container attach 0442176c73aac2f6f893aa94babe2c9b6df4e7af3893af496d45597d1e546a58 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-25T06:26:39.258 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:39 smithi175 podman[97747]: 2024-05-25 06:26:39.125284543 +0000 UTC m=+0.662431284 container died 0442176c73aac2f6f893aa94babe2c9b6df4e7af3893af496d45597d1e546a58 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-25T06:26:39.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:39 smithi179 ceph-mon[67580]: osdmap e110: 8 total, 7 up, 8 in 2024-05-25T06:26:39.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:39 smithi179 ceph-mon[67580]: pgmap v71: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:39.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:39 smithi175 ceph-mon[84648]: osdmap e110: 8 total, 7 up, 8 in 2024-05-25T06:26:39.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:39 smithi175 ceph-mon[84648]: pgmap v71: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:39.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:39 smithi175 ceph-mon[89733]: osdmap e110: 8 total, 7 up, 8 in 2024-05-25T06:26:39.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:39 smithi175 ceph-mon[89733]: pgmap v71: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:26:40.070 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:39 smithi175 podman[97789]: 2024-05-25 06:26:39.680651472 +0000 UTC m=+0.549679900 container remove 0442176c73aac2f6f893aa94babe2c9b6df4e7af3893af496d45597d1e546a58 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-25T06:26:40.070 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:39 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.0.service: Deactivated successfully. 2024-05-25T06:26:40.070 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:39 smithi175 systemd[1]: Stopped Ceph osd.0 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:26:40.070 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:39 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.0.service: Consumed 14.261s CPU time. 2024-05-25T06:26:40.070 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:39 smithi175 systemd[1]: Starting Ceph osd.0 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:26:40.375 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 podman[97889]: 2024-05-25 06:26:40.022193059 +0000 UTC m=+0.017083162 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:26:40.375 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 podman[97889]: 2024-05-25 06:26:40.139950273 +0000 UTC m=+0.134840358 container create f6f15f1720fd5be3bdc89c9cb900f02078372ed0cb5a928a3e231a9a91e4b0b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True) 2024-05-25T06:26:40.820 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 podman[97889]: 2024-05-25 06:26:40.376049104 +0000 UTC m=+0.370939216 container init f6f15f1720fd5be3bdc89c9cb900f02078372ed0cb5a928a3e231a9a91e4b0b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-05-25T06:26:40.820 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 podman[97889]: 2024-05-25 06:26:40.379826357 +0000 UTC m=+0.374716452 container start f6f15f1720fd5be3bdc89c9cb900f02078372ed0cb5a928a3e231a9a91e4b0b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-05-25T06:26:40.820 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 podman[97889]: 2024-05-25 06:26:40.459249606 +0000 UTC m=+0.454139701 container attach f6f15f1720fd5be3bdc89c9cb900f02078372ed0cb5a928a3e231a9a91e4b0b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521) 2024-05-25T06:26:41.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate[97904]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-25T06:26:41.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate[97904]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-25T06:26:41.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 bash[97889]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-25T06:26:41.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 bash[97889]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-25T06:26:41.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 bash[97889]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-25T06:26:41.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 bash[97889]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-25T06:26:41.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 bash[97889]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 bash[97889]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 bash[97889]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate[97904]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate[97904]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate[97904]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate[97904]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate[97904]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-25T06:26:41.321 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:40 smithi175 podman[97889]: 2024-05-25 06:26:40.854811202 +0000 UTC m=+0.849701297 container died f6f15f1720fd5be3bdc89c9cb900f02078372ed0cb5a928a3e231a9a91e4b0b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, GIT_CLEAN=True, ceph=True) 2024-05-25T06:26:41.629 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:41 smithi175 ceph-mon[89733]: pgmap v72: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:26:41.630 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:41 smithi175 ceph-mon[84648]: pgmap v72: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:26:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:41 smithi179 ceph-mon[67580]: pgmap v72: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:26:41.961 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:41 smithi175 podman[98027]: 2024-05-25 06:26:41.63012548 +0000 UTC m=+0.769641999 container remove f6f15f1720fd5be3bdc89c9cb900f02078372ed0cb5a928a3e231a9a91e4b0b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD) 2024-05-25T06:26:41.961 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:41 smithi175 podman[98088]: 2024-05-25 06:26:41.839329283 +0000 UTC m=+0.106025448 container create e55ca2bd159e08c4d177ac6392ee450cabf6314501ddd917429d241496aba784 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-05-25T06:26:41.961 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:41 smithi175 podman[98088]: 2024-05-25 06:26:41.750792979 +0000 UTC m=+0.017489153 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:26:42.319 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:42 smithi175 podman[98088]: 2024-05-25 06:26:42.041923632 +0000 UTC m=+0.308619808 container init e55ca2bd159e08c4d177ac6392ee450cabf6314501ddd917429d241496aba784 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, ceph=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:26:42.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:42 smithi175 podman[98088]: 2024-05-25 06:26:42.045415798 +0000 UTC m=+0.312111963 container start e55ca2bd159e08c4d177ac6392ee450cabf6314501ddd917429d241496aba784 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, ceph=True) 2024-05-25T06:26:42.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:42 smithi175 ceph-osd[98107]: -- 172.21.15.175:0/4202623819 <== mon.0 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563f88f77680 con 0x563f88f41400 2024-05-25T06:26:42.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:42 smithi175 bash[98088]: e55ca2bd159e08c4d177ac6392ee450cabf6314501ddd917429d241496aba784 2024-05-25T06:26:42.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:42 smithi175 systemd[1]: Started Ceph osd.0 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:26:43.039 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:42 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[98103]: 2024-05-25T06:26:42.592+0000 7f907c2827c0 -1 Falling back to public interface 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[84648]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:43.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:43 smithi175 ceph-mon[89733]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:43 smithi179 ceph-mon[67580]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:26:43] "GET /metrics HTTP/1.1" 200 38019 "" "Prometheus/2.43.0" 2024-05-25T06:26:44.453 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:44 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded (PG_DEGRADED) 2024-05-25T06:26:44.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:44 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded (PG_DEGRADED) 2024-05-25T06:26:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:44 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded (PG_DEGRADED) 2024-05-25T06:26:45.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:45 smithi179 ceph-mon[67580]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:45.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:45 smithi175 ceph-mon[84648]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:45.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:45 smithi175 ceph-mon[89733]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:45.571 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[98103]: 2024-05-25T06:26:45.299+0000 7f907c2827c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-05-25T06:26:45.571 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:45 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[98103]: 2024-05-25T06:26:45.325+0000 7f907c2827c0 -1 osd.0 108 log_to_monitors true 2024-05-25T06:26:46.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:46 smithi179 ceph-mon[67580]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T06:26:46.548 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:46 smithi175 ceph-mon[84648]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T06:26:46.549 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:46 smithi175 ceph-mon[89733]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-25T06:26:46.819 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:26:46 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[98103]: 2024-05-25T06:26:46.659+0000 7f90696d1700 -1 osd.0 108 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:26:47.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:47 smithi179 ceph-mon[67580]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-25T06:26:47.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:47 smithi179 ceph-mon[67580]: osdmap e111: 8 total, 7 up, 8 in 2024-05-25T06:26:47.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:47 smithi179 ceph-mon[67580]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:26:47.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:47 smithi179 ceph-mon[67580]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:47.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:47 smithi179 ceph-mon[67580]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' 2024-05-25T06:26:47.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:47.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:47.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[84648]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[84648]: osdmap e111: 8 total, 7 up, 8 in 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[84648]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[84648]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[84648]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[89733]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[89733]: osdmap e111: 8 total, 7 up, 8 in 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[89733]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:26:47.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[89733]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:47.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[89733]: from='osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341]' entity='osd.0' 2024-05-25T06:26:47.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:47.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:48.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:48 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:48.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:48 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:26:48.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:48 smithi179 ceph-mon[67580]: osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341] boot 2024-05-25T06:26:48.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:48 smithi179 ceph-mon[67580]: osdmap e112: 8 total, 8 up, 8 in 2024-05-25T06:26:48.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:26:48.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:48.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:48.564 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[89733]: osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341] boot 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[89733]: osdmap e112: 8 total, 8 up, 8 in 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[84648]: osd.0 [v2:172.21.15.175:6802/3287322341,v1:172.21.15.175:6803/3287322341] boot 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[84648]: osdmap e112: 8 total, 8 up, 8 in 2024-05-25T06:26:48.565 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-25T06:26:48.566 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:48.566 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:49 smithi179 ceph-mon[67580]: pgmap v78: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:49 smithi179 ceph-mon[67580]: osdmap e113: 8 total, 8 up, 8 in 2024-05-25T06:26:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:49 smithi175 ceph-mon[89733]: pgmap v78: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:49 smithi175 ceph-mon[89733]: osdmap e113: 8 total, 8 up, 8 in 2024-05-25T06:26:49.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:49 smithi175 ceph-mon[84648]: pgmap v78: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-25T06:26:49.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:49 smithi175 ceph-mon[84648]: osdmap e113: 8 total, 8 up, 8 in 2024-05-25T06:26:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:51 smithi179 ceph-mon[67580]: pgmap v80: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-25T06:26:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:51.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:51 smithi175 ceph-mon[84648]: pgmap v80: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-25T06:26:51.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:51 smithi175 ceph-mon[89733]: pgmap v80: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-25T06:26:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:26:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:26:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:26:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:26:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:26:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:26:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:53.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:52 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:26:53.063 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:26:53.064 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:26:53.065 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:53.065 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:52 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:26:53.778 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:26:53] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.43.0" 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:26:53.644+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:26:53.779 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:53.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:53.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:26:53.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:53.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:53.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:53.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:53.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:26:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:53 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded) 2024-05-25T06:26:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-25T06:26:54.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded) 2024-05-25T06:26:54.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 76/723 objects degraded (10.512%), 20 pgs degraded) 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:26:54.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-25T06:26:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:54 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:55.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:55.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:55.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:55.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:55.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:54 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:54 smithi179 ceph-mon[67580]: Upgrade: osd.1 is safe to restart 2024-05-25T06:26:55.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:54 smithi179 ceph-mon[67580]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 0 op/s 2024-05-25T06:26:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:55.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:55.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:55.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:55.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:55.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:55.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:26:55.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:26:55.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[84648]: Upgrade: osd.1 is safe to restart 2024-05-25T06:26:55.078 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[84648]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 0 op/s 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[89733]: Upgrade: osd.1 is safe to restart 2024-05-25T06:26:55.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:54 smithi175 ceph-mon[89733]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 0 op/s 2024-05-25T06:26:56.438 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[89733]: Upgrade: Updating osd.1 2024-05-25T06:26:56.438 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[89733]: Deploying daemon osd.1 on smithi175 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[84648]: Upgrade: Updating osd.1 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:56.439 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:56 smithi175 ceph-mon[84648]: Deploying daemon osd.1 on smithi175 2024-05-25T06:26:56.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:56 smithi179 ceph-mon[67580]: Upgrade: Updating osd.1 2024-05-25T06:26:56.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:26:56.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-25T06:26:56.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:26:56.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:56 smithi179 ceph-mon[67580]: Deploying daemon osd.1 on smithi175 2024-05-25T06:26:57.491 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:57 smithi175 ceph-mon[89733]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-05-25T06:26:57.491 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:57 smithi175 ceph-mon[84648]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-05-25T06:26:57.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:57 smithi179 ceph-mon[67580]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-05-25T06:26:58.490 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:58 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:58.490 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:58 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:58.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:58 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:26:58.746 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:26:58 smithi175 systemd[1]: Stopping Ceph osd.1 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:26:59.069 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:26:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[35225]: 2024-05-25T06:26:58.745+0000 7f598ec13700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:26:59.070 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:26:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[35225]: 2024-05-25T06:26:58.745+0000 7f598ec13700 -1 osd.1 113 *** Got signal Terminated *** 2024-05-25T06:26:59.070 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:26:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[35225]: 2024-05-25T06:26:58.745+0000 7f598ec13700 -1 osd.1 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:26:59.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:59 smithi179 ceph-mon[67580]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:26:59.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:26:59 smithi179 ceph-mon[67580]: osd.1 marked itself down and dead 2024-05-25T06:26:59.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:59 smithi175 ceph-mon[89733]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:26:59.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:26:59 smithi175 ceph-mon[89733]: osd.1 marked itself down and dead 2024-05-25T06:26:59.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:59 smithi175 ceph-mon[84648]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:26:59.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:26:59 smithi175 ceph-mon[84648]: osd.1 marked itself down and dead 2024-05-25T06:26:59.747 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:27:00.069 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:26:59 smithi175 podman[102507]: 2024-05-25 06:26:59.798395898 +0000 UTC m=+1.160769635 container died cf7becc5812d7ef4d76593b6c7a12c79bb0cb121c1c49ff888f14e36996214d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.openshift.expose-services=, distribution-scope=public, ceph=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.tags=base centos centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-25T06:27:00.427 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:00 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:27:00.428 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:00 smithi175 ceph-mon[89733]: osdmap e114: 8 total, 7 up, 8 in 2024-05-25T06:27:00.428 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:00 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:27:00.428 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:00 smithi175 ceph-mon[84648]: osdmap e114: 8 total, 7 up, 8 in 2024-05-25T06:27:00.428 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:00 smithi175 podman[102507]: 2024-05-25 06:27:00.185162916 +0000 UTC m=+1.547536645 container cleanup cf7becc5812d7ef4d76593b6c7a12c79bb0cb121c1c49ff888f14e36996214d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, RELEASE=HEAD, architecture=x86_64, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., release=754, GIT_CLEAN=True, vcs-type=git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-25T06:27:00.429 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:00 smithi175 bash[102507]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1 2024-05-25T06:27:00.448 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:27:00.448 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (5m) 13s ago 27m 28.8M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:27:00.448 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (4m) 2m ago 26m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 13s ago 26m 62.5M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (6m) 2m ago 31m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (16m) 13s ago 33m 506M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (2m) 13s ago 33m 45.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (3m) 2m ago 31m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (2m) 13s ago 32m 39.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (5m) 13s ago 27m 15.8M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (5m) 2m ago 27m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (18s) 13s ago 31m 12.6M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (30m) 13s ago 30m 59.6M 2992M 17.2.0 e1d6a67b021e cf7becc5812d 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (30m) 13s ago 30m 54.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (30m) 13s ago 30m 54.1M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (29m) 2m ago 29m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:27:00.449 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (29m) 2m ago 29m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:27:00.450 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (29m) 2m ago 29m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:27:00.450 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (28m) 2m ago 28m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:27:00.450 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (5m) 2m ago 28m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:27:00.450 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (26m) 13s ago 26m 77.7M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:27:00.450 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (26m) 2m ago 26m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:27:00.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:00 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:27:00.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:00 smithi179 ceph-mon[67580]: osdmap e114: 8 total, 7 up, 8 in 2024-05-25T06:27:00.804 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:00 smithi175 podman[102551]: 2024-05-25 06:27:00.428565726 +0000 UTC m=+0.623466818 container remove cf7becc5812d7ef4d76593b6c7a12c79bb0cb121c1c49ff888f14e36996214d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, ceph=True, distribution-scope=public, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, release=754, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, name=centos-stream, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.buildah.version=1.19.8) 2024-05-25T06:27:00.889 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:27:00.889 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:27:00.889 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:27:00.889 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 1 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 6 2024-05-25T06:27:00.890 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:27:00.891 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:27:01.070 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:00 smithi175 podman[102706]: 2024-05-25 06:27:00.70820364 +0000 UTC m=+0.017475019 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:27:01.070 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:00 smithi175 podman[102706]: 2024-05-25 06:27:00.837961179 +0000 UTC m=+0.147232558 container create 98e92f1abb67db79dd8e3be1802132ac5c67b8f425277ac3545a29d02f8f3727 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-25T06:27:01.232 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:27:01.232 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:27:01.232 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:27:01.232 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-05-25T06:27:01.232 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:27:01.233 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "1/8 daemons upgraded", 2024-05-25T06:27:01.233 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading osd daemons", 2024-05-25T06:27:01.233 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:27:01.233 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:27:01.401 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[89733]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.401 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[89733]: pgmap v87: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:27:01.401 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[89733]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.401 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[89733]: osdmap e115: 8 total, 7 up, 8 in 2024-05-25T06:27:01.401 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[89733]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.401 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:01.401 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2405584488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:01.401 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:01 smithi175 podman[102706]: 2024-05-25 06:27:01.090729686 +0000 UTC m=+0.400001066 container init 98e92f1abb67db79dd8e3be1802132ac5c67b8f425277ac3545a29d02f8f3727 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-05-25T06:27:01.402 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:01 smithi175 podman[102706]: 2024-05-25 06:27:01.094394036 +0000 UTC m=+0.403665425 container start 98e92f1abb67db79dd8e3be1802132ac5c67b8f425277ac3545a29d02f8f3727 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:27:01.402 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:01 smithi175 podman[102706]: 2024-05-25 06:27:01.165691334 +0000 UTC m=+0.474962726 container attach 98e92f1abb67db79dd8e3be1802132ac5c67b8f425277ac3545a29d02f8f3727 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-05-25T06:27:01.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[84648]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[84648]: pgmap v87: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:27:01.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[84648]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[84648]: osdmap e115: 8 total, 7 up, 8 in 2024-05-25T06:27:01.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[84648]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:01.402 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:01 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2405584488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:01 smithi179 ceph-mon[67580]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:01 smithi179 ceph-mon[67580]: pgmap v87: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:27:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:01 smithi179 ceph-mon[67580]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:01 smithi179 ceph-mon[67580]: osdmap e115: 8 total, 7 up, 8 in 2024-05-25T06:27:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:01 smithi179 ceph-mon[67580]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:01 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:01.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:01 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2405584488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:01.685 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:01 smithi175 podman[102706]: 2024-05-25 06:27:01.401619496 +0000 UTC m=+0.710890879 container died 98e92f1abb67db79dd8e3be1802132ac5c67b8f425277ac3545a29d02f8f3727 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, ceph=True) 2024-05-25T06:27:02.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:02 smithi179 ceph-mon[67580]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:02.545 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:02 smithi175 ceph-mon[84648]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:02.546 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:02 smithi175 ceph-mon[89733]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:02.546 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 podman[102757]: 2024-05-25 06:27:02.10339135 +0000 UTC m=+0.695873013 container remove 98e92f1abb67db79dd8e3be1802132ac5c67b8f425277ac3545a29d02f8f3727 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, ceph=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:27:02.546 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.1.service: Deactivated successfully. 2024-05-25T06:27:02.546 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 systemd[1]: Stopped Ceph osd.1 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:27:02.546 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.1.service: Consumed 14.853s CPU time. 2024-05-25T06:27:02.546 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 systemd[1]: Starting Ceph osd.1 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:27:02.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 podman[102852]: 2024-05-25 06:27:02.545998561 +0000 UTC m=+0.112393725 container create 09125b11e1d6dd1910ae6d5631ce28e948f94fd26cc7b27e3159e1e4d3dbd424 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, ceph=True) 2024-05-25T06:27:02.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 podman[102852]: 2024-05-25 06:27:02.450992442 +0000 UTC m=+0.017387583 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:27:02.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 podman[102852]: 2024-05-25 06:27:02.756686388 +0000 UTC m=+0.323081539 container init 09125b11e1d6dd1910ae6d5631ce28e948f94fd26cc7b27e3159e1e4d3dbd424 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-25T06:27:02.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 podman[102852]: 2024-05-25 06:27:02.760236709 +0000 UTC m=+0.326631851 container start 09125b11e1d6dd1910ae6d5631ce28e948f94fd26cc7b27e3159e1e4d3dbd424 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-25T06:27:03.216 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:02 smithi175 podman[102852]: 2024-05-25 06:27:02.84632473 +0000 UTC m=+0.412719889 container attach 09125b11e1d6dd1910ae6d5631ce28e948f94fd26cc7b27e3159e1e4d3dbd424 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-25T06:27:03.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:03 smithi179 ceph-mon[67580]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:03.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:03 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 100/723 objects degraded (13.831%), 20 pgs degraded (PG_DEGRADED) 2024-05-25T06:27:03.551 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:03 smithi175 ceph-mon[89733]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:03.551 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:03 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 100/723 objects degraded (13.831%), 20 pgs degraded (PG_DEGRADED) 2024-05-25T06:27:03.551 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:03 smithi175 ceph-mon[84648]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:03.552 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:03 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 100/723 objects degraded (13.831%), 20 pgs degraded (PG_DEGRADED) 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate[102866]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate[102866]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 bash[102852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 bash[102852]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 bash[102852]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 bash[102852]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 bash[102852]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-25T06:27:03.552 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 bash[102852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-25T06:27:03.553 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 bash[102852]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-25T06:27:03.553 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate[102866]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-25T06:27:03.553 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate[102866]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-25T06:27:03.553 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate[102866]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-25T06:27:03.553 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate[102866]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-25T06:27:03.553 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate[102866]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-25T06:27:03.553 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 podman[102852]: 2024-05-25 06:27:03.242285671 +0000 UTC m=+0.808680813 container died 09125b11e1d6dd1910ae6d5631ce28e948f94fd26cc7b27e3159e1e4d3dbd424 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:27:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:27:03] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.43.0" 2024-05-25T06:27:04.178 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:03 smithi175 podman[102983]: 2024-05-25 06:27:03.978432944 +0000 UTC m=+0.730322775 container remove 09125b11e1d6dd1910ae6d5631ce28e948f94fd26cc7b27e3159e1e4d3dbd424 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-25T06:27:04.497 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 podman[103059]: 2024-05-25 06:27:04.179127499 +0000 UTC m=+0.102888244 container create 3d00046530798dd980198f4e03204ac5714087b21fcd3af56318ef99c0d079ea (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-25T06:27:04.498 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 podman[103059]: 2024-05-25 06:27:04.093710985 +0000 UTC m=+0.017471741 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:27:04.498 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 podman[103059]: 2024-05-25 06:27:04.389804552 +0000 UTC m=+0.313565308 container init 3d00046530798dd980198f4e03204ac5714087b21fcd3af56318ef99c0d079ea (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-25T06:27:04.498 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 podman[103059]: 2024-05-25 06:27:04.393122787 +0000 UTC m=+0.316883542 container start 3d00046530798dd980198f4e03204ac5714087b21fcd3af56318ef99c0d079ea (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:27:04.498 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 ceph-osd[103079]: -- 172.21.15.175:0/3350753526 <== mon.1 v2:172.21.15.175:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cf68c99680 con 0x55cf68c63000 2024-05-25T06:27:04.498 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 bash[103059]: 3d00046530798dd980198f4e03204ac5714087b21fcd3af56318ef99c0d079ea 2024-05-25T06:27:04.819 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 systemd[1]: Started Ceph osd.1 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:27:05.262 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[103075]: 2024-05-25T06:27:04.931+0000 7f567f31e7c0 -1 Falling back to public interface 2024-05-25T06:27:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:05 smithi179 ceph-mon[67580]: pgmap v90: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[89733]: pgmap v90: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[84648]: pgmap v90: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:05.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:06.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:07.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:07 smithi179 ceph-mon[67580]: pgmap v91: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:07.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:07 smithi175 ceph-mon[84648]: pgmap v91: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:07 smithi175 ceph-mon[89733]: pgmap v91: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:07.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[103075]: 2024-05-25T06:27:07.651+0000 7f567f31e7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-05-25T06:27:07.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[103075]: 2024-05-25T06:27:07.688+0000 7f567f31e7c0 -1 osd.1 113 log_to_monitors true 2024-05-25T06:27:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:08 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:08 smithi179 ceph-mon[67580]: from='osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T06:27:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:08 smithi179 ceph-mon[67580]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T06:27:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:08 smithi179 ceph-mon[67580]: pgmap v92: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[84648]: from='osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[84648]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[84648]: pgmap v92: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[89733]: from='osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[89733]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-25T06:27:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:08 smithi175 ceph-mon[89733]: pgmap v92: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:09.319 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:27:08 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[103075]: 2024-05-25T06:27:08.967+0000 7f566c76d700 -1 osd.1 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:27:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:09 smithi179 ceph-mon[67580]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-25T06:27:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:09 smithi179 ceph-mon[67580]: osdmap e116: 8 total, 7 up, 8 in 2024-05-25T06:27:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:09 smithi179 ceph-mon[67580]: from='osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:27:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:09 smithi179 ceph-mon[67580]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:27:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:09 smithi179 ceph-mon[67580]: from='osd.1 ' entity='osd.1' 2024-05-25T06:27:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[89733]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[89733]: osdmap e116: 8 total, 7 up, 8 in 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[89733]: from='osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[89733]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[89733]: from='osd.1 ' entity='osd.1' 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[84648]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-25T06:27:09.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[84648]: osdmap e116: 8 total, 7 up, 8 in 2024-05-25T06:27:09.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[84648]: from='osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:27:09.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[84648]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:27:09.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[84648]: from='osd.1 ' entity='osd.1' 2024-05-25T06:27:09.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:09.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:10 smithi179 ceph-mon[67580]: pgmap v94: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:10 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:27:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:10 smithi179 ceph-mon[67580]: osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023] boot 2024-05-25T06:27:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:10 smithi179 ceph-mon[67580]: osdmap e117: 8 total, 8 up, 8 in 2024-05-25T06:27:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:10 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:27:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:10 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:10.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:10 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[84648]: pgmap v94: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[84648]: osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023] boot 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[84648]: osdmap e117: 8 total, 8 up, 8 in 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[89733]: pgmap v94: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[89733]: osd.1 [v2:172.21.15.175:6810/735637023,v1:172.21.15.175:6811/735637023] boot 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[89733]: osdmap e117: 8 total, 8 up, 8 in 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-25T06:27:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:10.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:10 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:12.141 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:11 smithi175 ceph-mon[89733]: osdmap e118: 8 total, 8 up, 8 in 2024-05-25T06:27:12.142 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:11 smithi175 ceph-mon[84648]: osdmap e118: 8 total, 8 up, 8 in 2024-05-25T06:27:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:11 smithi179 ceph-mon[67580]: osdmap e118: 8 total, 8 up, 8 in 2024-05-25T06:27:13.264 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:12 smithi175 ceph-mon[89733]: pgmap v97: 161 pgs: 6 active+undersized+degraded, 15 active+undersized, 140 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-05-25T06:27:13.264 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:12 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded (PG_DEGRADED) 2024-05-25T06:27:13.264 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:12 smithi175 ceph-mon[84648]: pgmap v97: 161 pgs: 6 active+undersized+degraded, 15 active+undersized, 140 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-05-25T06:27:13.264 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:12 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded (PG_DEGRADED) 2024-05-25T06:27:13.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:12 smithi179 ceph-mon[67580]: pgmap v97: 161 pgs: 6 active+undersized+degraded, 15 active+undersized, 140 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-05-25T06:27:13.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:12 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded (PG_DEGRADED) 2024-05-25T06:27:13.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:27:13] "GET /metrics HTTP/1.1" 200 38175 "" "Prometheus/2.43.0" 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:14.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:14.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:14.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:14 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-05-25T06:27:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-05-25T06:27:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:15.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:14 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:15.773 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:27:15.585+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:15.773 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:15.773 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:15.773 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:15.774 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:15.775 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:15.776 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:15.777 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:15.778 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:15.778 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:15.778 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:15.778 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:15.778 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:27:16.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:16.038 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.072 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:27:16.073 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.075 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:16.076 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:16.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:16.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:16.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:16.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:16.825 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:16.826 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:16.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: Upgrade: Finalizing container_image settings 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: Upgrade: Complete! 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 876 B/s rd, 0 op/s 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:16.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: Upgrade: Finalizing container_image settings 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: Upgrade: Complete! 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 876 B/s rd, 0 op/s 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-05-25T06:27:16.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:16.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:16.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:16.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:16 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:17.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:17.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:17.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:17.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: Upgrade: Finalizing container_image settings 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: Upgrade: Complete! 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 876 B/s rd, 0 op/s 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:17.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:16 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:17.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:27:17.645+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:17.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:17.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:17.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:17.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:17.781 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:17.782 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:17.783 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:17.784 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:17.785 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:17.786 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:17.786 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:17 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:17 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:17 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:17 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:18.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:17 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:18.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:17 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:18.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:18.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:18.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:17 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:27:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:19.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:19.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:18 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:19.071 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:19.072 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.073 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.075 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:19.076 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.077 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-05-25T06:27:19.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:27:19.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:19.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:19.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:19.080 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:18 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:19.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:27:19.557+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.821 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.822 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:27:19.823 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:19.824 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:19.825 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:19 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:21 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:27:21.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:21.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:21 smithi179 ceph-mon[67580]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-05-25T06:27:21.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:21.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:21.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.821 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:21.822 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.823 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:21.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:21.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:21.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:21.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:21.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:21.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:21.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:21.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:21.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:21.830 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:21.831 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:21.832 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:21.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:21.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:21.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:21.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.832 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[84648]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[89733]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:21.833 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:21.834 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:23 smithi179 ceph-mon[67580]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:27:23.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:23 smithi175 ceph-mon[84648]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:27:23.917 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:27:23] "GET /metrics HTTP/1.1" 200 38179 "" "Prometheus/2.43.0" 2024-05-25T06:27:23.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:23 smithi175 ceph-mon[89733]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:27:25.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:25 smithi179 ceph-mon[67580]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:25.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:25 smithi175 ceph-mon[84648]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:25.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:25 smithi175 ceph-mon[89733]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:26.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:26 smithi175 ceph-mon[84648]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-25T06:27:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:26 smithi175 ceph-mon[89733]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-25T06:27:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:26 smithi179 ceph-mon[67580]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-25T06:27:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:27 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:27 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:27 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:28.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:28 smithi175 ceph-mon[84648]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:27:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:28 smithi175 ceph-mon[89733]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:27:29.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:28 smithi179 ceph-mon[67580]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:27:31.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:30 smithi179 ceph-mon[67580]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:31.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:30 smithi175 ceph-mon[84648]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:31.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:30 smithi175 ceph-mon[89733]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:32.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:27:32.802 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:32 smithi175 ceph-mon[89733]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:32.802 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:32 smithi175 ceph-mon[89733]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:32.802 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:32 smithi175 ceph-mon[84648]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:32.803 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:32 smithi175 ceph-mon[84648]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:32 smithi179 ceph-mon[67580]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:32 smithi179 ceph-mon[67580]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:33.808 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:27:33] "GET /metrics HTTP/1.1" 200 38179 "" "Prometheus/2.43.0" 2024-05-25T06:27:34.240 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (5m) 25s ago 27m 28.8M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (5m) 2m ago 27m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 25s ago 26m 62.5M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (6m) 2m ago 32m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (16m) 25s ago 33m 508M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (3m) 25s ago 33m 47.8M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (3m) 2m ago 32m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (2m) 25s ago 32m 40.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (6m) 25s ago 28m 16.1M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (6m) 2m ago 28m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (52s) 25s ago 31m 51.5M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (30s) 25s ago 31m 12.2M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (31m) 25s ago 31m 55.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:27:34.241 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (30m) 25s ago 30m 54.7M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:27:34.242 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (30m) 2m ago 30m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:27:34.242 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (30m) 2m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:27:34.242 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (29m) 2m ago 29m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:27:34.242 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (29m) 2m ago 29m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:27:34.242 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (6m) 2m ago 28m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:27:34.242 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (27m) 25s ago 26m 77.8M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:27:34.242 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (26m) 2m ago 26m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:27:34.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:34 smithi175 ceph-mon[84648]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:34 smithi175 ceph-mon[84648]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:34 smithi175 ceph-mon[89733]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:34 smithi175 ceph-mon[89733]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:34.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-25T06:27:35.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:34 smithi179 ceph-mon[67580]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:35.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:34 smithi179 ceph-mon[67580]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:35.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:36.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:37.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:36 smithi179 ceph-mon[67580]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:37.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:36 smithi175 ceph-mon[84648]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:37.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:36 smithi175 ceph-mon[89733]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:37.261 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:27:37.844 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-05-25T06:27:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:37 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:38.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:37 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/749215223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:38.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:37 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:38.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:37 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/749215223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:38.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:37 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:38.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:37 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/749215223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:39.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:38 smithi179 ceph-mon[67580]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:39.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:38 smithi175 ceph-mon[84648]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:39.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:38 smithi175 ceph-mon[89733]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:41.012 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:40 smithi175 ceph-mon[89733]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:41.013 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:40 smithi175 ceph-mon[89733]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:41.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:40 smithi175 ceph-mon[84648]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:41.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:40 smithi175 ceph-mon[84648]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:40 smithi179 ceph-mon[67580]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:41.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:40 smithi179 ceph-mon[67580]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:42.048 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:27:42.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:27:43.021 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:42 smithi175 ceph-mon[84648]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:43.021 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:42 smithi175 ceph-mon[89733]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:43.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:42 smithi179 ceph-mon[67580]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:27:43] "GET /metrics HTTP/1.1" 200 38174 "" "Prometheus/2.43.0" 2024-05-25T06:27:44.661 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:27:44.661 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": null, 2024-05-25T06:27:44.661 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": false, 2024-05-25T06:27:44.661 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "", 2024-05-25T06:27:44.662 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:27:44.662 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": null, 2024-05-25T06:27:44.662 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "", 2024-05-25T06:27:44.662 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:27:44.662 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:27:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:44 smithi179 ceph-mon[67580]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:45.041 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:44 smithi175 ceph-mon[84648]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:45.041 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:44 smithi175 ceph-mon[89733]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:45.367 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:27:46.004 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:45 smithi175 ceph-mon[89733]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:46.004 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:45 smithi175 ceph-mon[84648]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:45 smithi179 ceph-mon[67580]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:46 smithi179 ceph-mon[67580]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:47.036 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:46 smithi175 ceph-mon[89733]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:47.037 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:46 smithi175 ceph-mon[84648]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:47.540 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:27:47.541 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:27:47.541 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:27:47.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:47 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:47.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:47 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/4025425033' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:27:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:47 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:47 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/4025425033' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:27:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:47 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:47 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/4025425033' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:27:48.200 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-05-25T06:27:49.032 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:48 smithi175 ceph-mon[89733]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:49.032 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:48 smithi175 ceph-mon[84648]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:48 smithi179 ceph-mon[67580]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:50 smithi179 ceph-mon[67580]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:50 smithi179 ceph-mon[67580]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:50 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:50 smithi175 ceph-mon[84648]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:50 smithi175 ceph-mon[84648]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:50 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:50 smithi175 ceph-mon[89733]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:50 smithi175 ceph-mon[89733]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:51.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:50 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:27:53.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:52 smithi179 ceph-mon[67580]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:53.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:52 smithi175 ceph-mon[89733]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:53.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:52 smithi175 ceph-mon[84648]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:53.438 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:27:53.800 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:27:53] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-05-25T06:27:54.115 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-25T06:27:54.778 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:27:54.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:54.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:54.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:54.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:54.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:54.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:54.779 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:54.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:27:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:27:55.716 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[89733]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[89733]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[89733]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[84648]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[84648]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[84648]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:55.717 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:55 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:55 smithi179 ceph-mon[67580]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:27:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:55 smithi179 ceph-mon[67580]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s 2024-05-25T06:27:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:55 smithi179 ceph-mon[67580]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:55 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:27:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:55 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:27:56.206 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:27:56.871 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:27:56.871 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (6m) 47s ago 28m 28.8M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (5m) 3m ago 27m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 47s ago 27m 62.5M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (7m) 3m ago 32m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (17m) 47s ago 34m 508M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (3m) 47s ago 34m 47.8M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (4m) 3m ago 32m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (3m) 47s ago 33m 40.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (6m) 47s ago 28m 16.1M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (6m) 3m ago 28m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (75s) 47s ago 32m 51.5M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (52s) 47s ago 31m 12.2M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (31m) 47s ago 31m 55.6M 2992M 17.2.0 e1d6a67b021e 5b8c6944f1e8 2024-05-25T06:27:56.872 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (31m) 47s ago 31m 54.7M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:27:56.873 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (30m) 3m ago 30m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:27:56.873 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (30m) 3m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:27:56.873 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (29m) 3m ago 29m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:27:56.873 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (29m) 3m ago 29m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:27:56.873 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (6m) 3m ago 28m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:27:56.873 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (27m) 47s ago 27m 77.8M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:27:56.873 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (27m) 3m ago 27m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:27:57.278 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:27:57.279 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:27:57.280 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:27:57.280 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-25T06:27:57.280 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 7 2024-05-25T06:27:57.280 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:27:57.280 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:27:57.278+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:57.445 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:57.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:57.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:57.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:57.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:57.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:57.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:57.446 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:57.447 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:27:57.448 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:57.449 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:27:57 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "", 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 image", 2024-05-25T06:27:57.647 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:27:57.648 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:27:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:57 smithi179 ceph-mon[67580]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:27:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:57 smithi179 ceph-mon[67580]: from='client.44358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:57 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2059912543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:57.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:57 smithi175 ceph-mon[84648]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:27:57.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:57 smithi175 ceph-mon[84648]: from='client.44358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:57 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2059912543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:57 smithi175 ceph-mon[89733]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:27:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:57 smithi175 ceph-mon[89733]: from='client.44358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:57 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2059912543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:27:58.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[84648]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:58.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:58.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[84648]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:58.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:58.570 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:58.571 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:58.572 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.573 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[89733]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:58.574 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[89733]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:58.575 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:58.576 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:58.577 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:58.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[89733]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.580 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:58 smithi175 ceph-mon[84648]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:27:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:58 smithi179 ceph-mon[67580]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:58 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:58 smithi179 ceph-mon[67580]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:58 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:27:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:27:58.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:27:58.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:27:58.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:27:58.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:27:58.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:27:58.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:58 smithi179 ceph-mon[67580]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:27:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:59 smithi179 ceph-mon[67580]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:27:59 smithi179 ceph-mon[67580]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:59.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:59 smithi175 ceph-mon[89733]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:59.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:27:59 smithi175 ceph-mon[89733]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:27:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:59 smithi175 ceph-mon[84648]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:27:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:27:59 smithi175 ceph-mon[84648]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:00.692 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.692 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mgr 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mon 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all crash 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-25T06:28:00.693 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[84648]: Upgrade: osd.2 is safe to restart 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mgr 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mon 2024-05-25T06:28:00.694 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all crash 2024-05-25T06:28:00.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-25T06:28:00.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-25T06:28:00.695 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:00 smithi175 ceph-mon[89733]: Upgrade: osd.2 is safe to restart 2024-05-25T06:28:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:28:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mgr 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mon 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:00.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all crash 2024-05-25T06:28:00.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:00.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-25T06:28:00.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-25T06:28:00.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:00 smithi179 ceph-mon[67580]: Upgrade: osd.2 is safe to restart 2024-05-25T06:28:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:01 smithi179 ceph-mon[67580]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:28:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:01 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:01 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T06:28:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:01 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:01.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[84648]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:28:01.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T06:28:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[89733]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-05-25T06:28:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-25T06:28:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:01 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:02.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:02 smithi175 ceph-mon[84648]: Upgrade: Updating osd.2 2024-05-25T06:28:02.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:02 smithi175 ceph-mon[84648]: Deploying daemon osd.2 on smithi175 2024-05-25T06:28:02.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:02 smithi175 ceph-mon[89733]: Upgrade: Updating osd.2 2024-05-25T06:28:02.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:02 smithi175 ceph-mon[89733]: Deploying daemon osd.2 on smithi175 2024-05-25T06:28:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:02 smithi179 ceph-mon[67580]: Upgrade: Updating osd.2 2024-05-25T06:28:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:02 smithi179 ceph-mon[67580]: Deploying daemon osd.2 on smithi175 2024-05-25T06:28:03.555 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:03 smithi175 ceph-mon[84648]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:03.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:03 smithi175 ceph-mon[89733]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:03 smithi179 ceph-mon[67580]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:03.817 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:28:03] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-05-25T06:28:04.493 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:04 smithi175 systemd[1]: Stopping Ceph osd.2 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:28:05.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[38128]: 2024-05-25T06:28:04.746+0000 7f4dc3707700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:28:05.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[38128]: 2024-05-25T06:28:04.746+0000 7f4dc3707700 -1 osd.2 118 *** Got signal Terminated *** 2024-05-25T06:28:05.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[38128]: 2024-05-25T06:28:04.746+0000 7f4dc3707700 -1 osd.2 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:28:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:05 smithi179 ceph-mon[67580]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:05 smithi179 ceph-mon[67580]: osd.2 marked itself down and dead 2024-05-25T06:28:05.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:05 smithi175 ceph-mon[84648]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:05.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:05 smithi175 ceph-mon[84648]: osd.2 marked itself down and dead 2024-05-25T06:28:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:05 smithi175 ceph-mon[89733]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:05 smithi175 ceph-mon[89733]: osd.2 marked itself down and dead 2024-05-25T06:28:06.172 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:05 smithi175 podman[109411]: 2024-05-25 06:28:05.861871781 +0000 UTC m=+1.222106107 container died 5b8c6944f1e81db47d2bb51cbdeb7276c6055887f62681dee0262f9948373563 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_CLEAN=True, maintainer=Guillaume Abrioux , distribution-scope=public, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc.) 2024-05-25T06:28:06.486 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:06 smithi175 podman[109411]: 2024-05-25 06:28:06.34681513 +0000 UTC m=+1.707049491 container cleanup 5b8c6944f1e81db47d2bb51cbdeb7276c6055887f62681dee0262f9948373563 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, build-date=2022-05-03T08:36:31.336870, release=754, distribution-scope=public, GIT_CLEAN=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., ceph=True, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream) 2024-05-25T06:28:06.486 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:06 smithi175 bash[109411]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2 2024-05-25T06:28:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:06 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:28:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:06 smithi179 ceph-mon[67580]: osdmap e119: 8 total, 7 up, 8 in 2024-05-25T06:28:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:06.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:06 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:28:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:06 smithi175 ceph-mon[89733]: osdmap e119: 8 total, 7 up, 8 in 2024-05-25T06:28:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:06 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:28:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:06 smithi175 ceph-mon[84648]: osdmap e119: 8 total, 7 up, 8 in 2024-05-25T06:28:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:06.820 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:06 smithi175 podman[109425]: 2024-05-25 06:28:06.615200787 +0000 UTC m=+0.749132373 container remove 5b8c6944f1e81db47d2bb51cbdeb7276c6055887f62681dee0262f9948373563 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, architecture=x86_64, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, RELEASE=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., name=centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-25T06:28:07.301 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:06 smithi175 podman[109517]: 2024-05-25 06:28:06.875933114 +0000 UTC m=+0.017527243 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:28:07.302 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:06 smithi175 podman[109517]: 2024-05-25 06:28:06.974486167 +0000 UTC m=+0.116080292 container create 3cdbfc999e291fbe2f7885f1c056680699dee1874c0e944b85b48e05e271fc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-05-25T06:28:07.302 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:07 smithi175 podman[109517]: 2024-05-25 06:28:07.202659744 +0000 UTC m=+0.344253867 container init 3cdbfc999e291fbe2f7885f1c056680699dee1874c0e944b85b48e05e271fc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-05-25T06:28:07.302 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:07 smithi175 podman[109517]: 2024-05-25 06:28:07.20630222 +0000 UTC m=+0.347896335 container start 3cdbfc999e291fbe2f7885f1c056680699dee1874c0e944b85b48e05e271fc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-25T06:28:07.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:07 smithi175 ceph-mon[84648]: pgmap v131: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:07.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:07 smithi175 ceph-mon[84648]: osdmap e120: 8 total, 7 up, 8 in 2024-05-25T06:28:07.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:07 smithi175 ceph-mon[89733]: pgmap v131: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:07.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:07 smithi175 ceph-mon[89733]: osdmap e120: 8 total, 7 up, 8 in 2024-05-25T06:28:07.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:07 smithi175 podman[109517]: 2024-05-25 06:28:07.302087592 +0000 UTC m=+0.443681699 container attach 3cdbfc999e291fbe2f7885f1c056680699dee1874c0e944b85b48e05e271fc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD) 2024-05-25T06:28:07.571 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:07 smithi175 podman[109517]: 2024-05-25 06:28:07.511483291 +0000 UTC m=+0.653077409 container died 3cdbfc999e291fbe2f7885f1c056680699dee1874c0e944b85b48e05e271fc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, RELEASE=squid-ea0684c, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521) 2024-05-25T06:28:07.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:07 smithi179 ceph-mon[67580]: pgmap v131: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:07.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:07 smithi179 ceph-mon[67580]: osdmap e120: 8 total, 7 up, 8 in 2024-05-25T06:28:08.320 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 podman[109559]: 2024-05-25 06:28:08.064685846 +0000 UTC m=+0.542396101 container remove 3cdbfc999e291fbe2f7885f1c056680699dee1874c0e944b85b48e05e271fc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux ) 2024-05-25T06:28:08.320 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.2.service: Deactivated successfully. 2024-05-25T06:28:08.320 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 systemd[1]: Stopped Ceph osd.2 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:28:08.320 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.2.service: Consumed 12.336s CPU time. 2024-05-25T06:28:08.320 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 systemd[1]: Starting Ceph osd.2 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:28:08.616 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:08 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:08.617 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:08 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:08.617 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 podman[109659]: 2024-05-25 06:28:08.417720518 +0000 UTC m=+0.017704491 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:28:08.617 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 podman[109659]: 2024-05-25 06:28:08.515591718 +0000 UTC m=+0.115575675 container create fc1e83c6d8cbcaffcc728382ea69f7afddbcffa8370875b78ee8dcae1e97be2e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521) 2024-05-25T06:28:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:08 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:09.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 podman[109659]: 2024-05-25 06:28:08.727050626 +0000 UTC m=+0.327034595 container init fc1e83c6d8cbcaffcc728382ea69f7afddbcffa8370875b78ee8dcae1e97be2e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240521, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-25T06:28:09.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 podman[109659]: 2024-05-25 06:28:08.731052766 +0000 UTC m=+0.331036730 container start fc1e83c6d8cbcaffcc728382ea69f7afddbcffa8370875b78ee8dcae1e97be2e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0) 2024-05-25T06:28:09.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:08 smithi175 podman[109659]: 2024-05-25 06:28:08.810026028 +0000 UTC m=+0.410009995 container attach fc1e83c6d8cbcaffcc728382ea69f7afddbcffa8370875b78ee8dcae1e97be2e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate[109682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate[109682]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 bash[109659]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 bash[109659]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 bash[109659]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 bash[109659]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 bash[109659]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-25T06:28:09.501 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 bash[109659]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-25T06:28:09.502 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 bash[109659]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-25T06:28:09.502 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate[109682]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-25T06:28:09.502 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate[109682]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-25T06:28:09.502 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate[109682]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-25T06:28:09.502 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate[109682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-25T06:28:09.502 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate[109682]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-25T06:28:09.502 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 podman[109659]: 2024-05-25 06:28:09.21557497 +0000 UTC m=+0.815558942 container died fc1e83c6d8cbcaffcc728382ea69f7afddbcffa8370875b78ee8dcae1e97be2e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate, RELEASE=squid-ea0684c, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-25T06:28:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:09 smithi179 ceph-mon[67580]: pgmap v133: 161 pgs: 40 peering, 2 stale+active+clean, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:09 smithi179 ceph-mon[67580]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-05-25T06:28:09.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:09 smithi175 ceph-mon[84648]: pgmap v133: 161 pgs: 40 peering, 2 stale+active+clean, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:09.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:09 smithi175 ceph-mon[84648]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-05-25T06:28:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:09 smithi175 ceph-mon[89733]: pgmap v133: 161 pgs: 40 peering, 2 stale+active+clean, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:09.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:09 smithi175 ceph-mon[89733]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-05-25T06:28:10.237 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:09 smithi175 podman[109807]: 2024-05-25 06:28:09.914171073 +0000 UTC m=+0.692876698 container remove fc1e83c6d8cbcaffcc728382ea69f7afddbcffa8370875b78ee8dcae1e97be2e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:28:10.238 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 podman[109870]: 2024-05-25 06:28:10.031062765 +0000 UTC m=+0.017513048 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:28:10.238 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 podman[109870]: 2024-05-25 06:28:10.131832368 +0000 UTC m=+0.118282649 container create 29ab74e627e13978a8565a79055bdbbc6834c4653791ff485a0ad04bfb134f25 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521) 2024-05-25T06:28:10.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 podman[109870]: 2024-05-25 06:28:10.334119212 +0000 UTC m=+0.320569487 container init 29ab74e627e13978a8565a79055bdbbc6834c4653791ff485a0ad04bfb134f25 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-05-25T06:28:10.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 podman[109870]: 2024-05-25 06:28:10.337289196 +0000 UTC m=+0.323739461 container start 29ab74e627e13978a8565a79055bdbbc6834c4653791ff485a0ad04bfb134f25 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-25T06:28:10.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 ceph-osd[109889]: -- 172.21.15.175:0/1876986160 <== mon.0 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559545a9f680 con 0x559545a69000 2024-05-25T06:28:10.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 bash[109870]: 29ab74e627e13978a8565a79055bdbbc6834c4653791ff485a0ad04bfb134f25 2024-05-25T06:28:10.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 systemd[1]: Started Ceph osd.2 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:28:11.272 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:10 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[109885]: 2024-05-25T06:28:10.886+0000 7f0c925fd7c0 -1 Falling back to public interface 2024-05-25T06:28:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[84648]: pgmap v134: 161 pgs: 3 active+undersized+degraded, 40 peering, 4 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 24/723 objects degraded (3.320%) 2024-05-25T06:28:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 24/723 objects degraded (3.320%), 3 pgs degraded (PG_DEGRADED) 2024-05-25T06:28:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[89733]: pgmap v134: 161 pgs: 3 active+undersized+degraded, 40 peering, 4 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 24/723 objects degraded (3.320%) 2024-05-25T06:28:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 24/723 objects degraded (3.320%), 3 pgs degraded (PG_DEGRADED) 2024-05-25T06:28:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:11 smithi179 ceph-mon[67580]: pgmap v134: 161 pgs: 3 active+undersized+degraded, 40 peering, 4 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 24/723 objects degraded (3.320%) 2024-05-25T06:28:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:11 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 24/723 objects degraded (3.320%), 3 pgs degraded (PG_DEGRADED) 2024-05-25T06:28:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:12 smithi175 ceph-mon[84648]: pgmap v135: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:12 smithi175 ceph-mon[84648]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 10 pgs peering) 2024-05-25T06:28:12.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:12 smithi175 ceph-mon[89733]: pgmap v135: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:12.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:12 smithi175 ceph-mon[89733]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 10 pgs peering) 2024-05-25T06:28:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:12 smithi179 ceph-mon[67580]: pgmap v135: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:12 smithi179 ceph-mon[67580]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 10 pgs peering) 2024-05-25T06:28:13.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:28:13] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-05-25T06:28:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:13 smithi175 ceph-mon[89733]: from='osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T06:28:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:13 smithi175 ceph-mon[89733]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T06:28:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:13 smithi175 ceph-mon[84648]: from='osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T06:28:13.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:13 smithi175 ceph-mon[84648]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T06:28:13.821 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[109885]: 2024-05-25T06:28:13.624+0000 7f0c925fd7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-05-25T06:28:13.821 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[109885]: 2024-05-25T06:28:13.648+0000 7f0c925fd7c0 -1 osd.2 118 log_to_monitors true 2024-05-25T06:28:13.821 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:28:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[109885]: 2024-05-25T06:28:13.730+0000 7f0c83a54700 -1 osd.2 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:28:14.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:13 smithi179 ceph-mon[67580]: from='osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T06:28:14.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:13 smithi179 ceph-mon[67580]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-25T06:28:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:14 smithi179 ceph-mon[67580]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-25T06:28:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:14 smithi179 ceph-mon[67580]: osdmap e121: 8 total, 7 up, 8 in 2024-05-25T06:28:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:14 smithi179 ceph-mon[67580]: from='osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:28:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:14 smithi179 ceph-mon[67580]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:28:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:14 smithi179 ceph-mon[67580]: pgmap v137: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:15.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[84648]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-25T06:28:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[84648]: osdmap e121: 8 total, 7 up, 8 in 2024-05-25T06:28:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[84648]: from='osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:28:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[84648]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:28:15.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[84648]: pgmap v137: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[89733]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-25T06:28:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[89733]: osdmap e121: 8 total, 7 up, 8 in 2024-05-25T06:28:15.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[89733]: from='osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:28:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[89733]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:28:15.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:14 smithi175 ceph-mon[89733]: pgmap v137: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:15 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:28:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:15 smithi179 ceph-mon[67580]: osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099] boot 2024-05-25T06:28:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:15 smithi179 ceph-mon[67580]: osdmap e122: 8 total, 8 up, 8 in 2024-05-25T06:28:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:28:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[84648]: osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099] boot 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[84648]: osdmap e122: 8 total, 8 up, 8 in 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[89733]: osd.2 [v2:172.21.15.175:6818/1869942099,v1:172.21.15.175:6819/1869942099] boot 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[89733]: osdmap e122: 8 total, 8 up, 8 in 2024-05-25T06:28:16.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-25T06:28:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.872 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[89733]: osdmap e123: 8 total, 8 up, 8 in 2024-05-25T06:28:16.872 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.872 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.872 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[89733]: pgmap v140: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:16.872 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[84648]: osdmap e123: 8 total, 8 up, 8 in 2024-05-25T06:28:16.873 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.873 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:16.873 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:16 smithi175 ceph-mon[84648]: pgmap v140: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:16 smithi179 ceph-mon[67580]: osdmap e123: 8 total, 8 up, 8 in 2024-05-25T06:28:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:16 smithi179 ceph-mon[67580]: pgmap v140: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-25T06:28:18.196 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:18 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:18.196 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:18 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:28:18.196 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:18 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:18.196 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:18 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:28:18.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:18 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:18.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:18 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:28:19.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:19 smithi179 ceph-mon[67580]: pgmap v141: 161 pgs: 5 active+undersized+degraded, 12 active+undersized, 144 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 26/723 objects degraded (3.596%) 2024-05-25T06:28:19.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:19 smithi175 ceph-mon[89733]: pgmap v141: 161 pgs: 5 active+undersized+degraded, 12 active+undersized, 144 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 26/723 objects degraded (3.596%) 2024-05-25T06:28:19.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:19 smithi175 ceph-mon[84648]: pgmap v141: 161 pgs: 5 active+undersized+degraded, 12 active+undersized, 144 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 26/723 objects degraded (3.596%) 2024-05-25T06:28:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:21.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:21.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 831 B/s rd, 0 op/s 2024-05-25T06:28:21.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-05-25T06:28:21.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:21.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:21.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:21.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:21.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.287 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 831 B/s rd, 0 op/s 2024-05-25T06:28:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-05-25T06:28:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:21.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:21.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:21.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.322 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:21.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:21.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:21.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 831 B/s rd, 0 op/s 2024-05-25T06:28:21.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-05-25T06:28:21.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:21.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:21.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:21.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:21.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:21.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:21.917 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:21 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/723 objects degraded (3.596%), 5 pgs degraded) 2024-05-25T06:28:21.918 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:21 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/723 objects degraded (3.596%), 5 pgs degraded) 2024-05-25T06:28:22.261 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:28:21.915+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:22.262 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:22.263 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:28:22.264 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:22.265 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:22.266 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:22.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:21 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/723 objects degraded (3.596%), 5 pgs degraded) 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:22.997 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:22.998 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:22.999 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:23.000 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.001 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: Upgrade: Finalizing container_image settings 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:28:23.002 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.003 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.004 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:23.005 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:23.006 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.007 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: Upgrade: Complete! 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:28:23.008 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:23.009 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.010 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: Upgrade: Finalizing container_image settings 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:28:23.011 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: Upgrade: Complete! 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:28:23.012 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:23.013 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:22 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:23.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:28:23.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:23.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:23.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:23.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:23.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:23.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:23.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:23.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:23.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:23.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:23.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:23.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:23.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:23.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:23.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:23.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:23.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:23.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:23.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:23.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:23.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:23.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:23.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:23.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:23.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:23.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:28:23.291 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:23.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:28:23.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:23.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:28:23.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:23.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:23.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:23.292 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:23.293 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:28:23.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: Upgrade: Finalizing container_image settings 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:28:23.295 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:28:23.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:28:23.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:28:23.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:28:23.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:28:23.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.296 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: Upgrade: Complete! 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:23.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:23.299 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:22 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:23.815 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:28:23] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-05-25T06:28:24.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:24.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:24.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:23 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:28:23.877+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:24.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:24.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:28:24.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:24.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:24.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:23 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:23 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:23 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:24.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:23 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:24 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:25.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:25.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:28:25.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:24 smithi179 ceph-mon[67580]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-05-25T06:28:25.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:24 smithi179 ceph-mon[67580]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:25.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:24 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:25.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:24 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:25.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:24 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:25.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:24 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.320 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.321 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:25.322 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:25.323 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:25.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:25.325 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:25.326 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:25.327 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:25.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[89733]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[89733]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:25.329 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[84648]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[84648]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:25.330 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:24 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:26.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:28:25.940+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:26.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:26.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:26.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:26.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:26.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:26.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:26.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:26.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:26.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:25 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:26 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:26 smithi175 ceph-mon[89733]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:26 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:26.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:26.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:26.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:26.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:26 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:26 smithi175 ceph-mon[84648]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:26 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:26.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:26.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:26.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:26.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:26.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:26.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:26.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:26.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:26.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:26 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:27.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:26 smithi179 ceph-mon[67580]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:26 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:27.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:27.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:27.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:27.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:27.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:27.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:27.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:27 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:27.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:27 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:27 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:28.762 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:28 smithi175 ceph-mon[89733]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-05-25T06:28:28.762 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:28 smithi175 ceph-mon[89733]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:28.762 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:28 smithi175 ceph-mon[84648]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-05-25T06:28:28.762 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:28 smithi175 ceph-mon[84648]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:28.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:28:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:28 smithi179 ceph-mon[67580]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-05-25T06:28:29.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:28 smithi179 ceph-mon[67580]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:30.807 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (6m) 15s ago 28m 28.6M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (6m) 3m ago 28m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 15s ago 27m 62.6M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (7m) 3m ago 33m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (17m) 15s ago 34m 511M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (4m) 15s ago 34m 53.5M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (4m) 3m ago 33m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (3m) 15s ago 33m 46.4M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (7m) 15s ago 29m 16.5M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (7m) 3m ago 29m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (109s) 15s ago 32m 52.4M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (86s) 15s ago 32m 50.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:28:30.808 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (20s) 15s ago 32m 12.2M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (31m) 15s ago 31m 56.1M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (31m) 3m ago 31m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (30m) 3m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (30m) 3m ago 30m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (30m) 3m ago 30m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (6m) 3m ago 29m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (27m) 15s ago 27m 78.1M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:28:30.809 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (27m) 3m ago 27m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:28:31.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:30 smithi175 ceph-mon[89733]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-05-25T06:28:31.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:30 smithi175 ceph-mon[84648]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-05-25T06:28:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:30 smithi179 ceph-mon[67580]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-05-25T06:28:31.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-25T06:28:32.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:31 smithi175 ceph-mon[89733]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:32.254 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:31 smithi175 ceph-mon[84648]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:32.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:31 smithi179 ceph-mon[67580]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:33.278 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:33 smithi175 ceph-mon[84648]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:28:33.279 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:33 smithi175 ceph-mon[89733]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:28:33.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:33 smithi179 ceph-mon[67580]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:28:33.688 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:28:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:28:33] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-05-25T06:28:34.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:34 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2282081407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:34.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:34 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2282081407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:34.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:34 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2282081407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:34.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-05-25T06:28:35.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:35 smithi179 ceph-mon[67580]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:35.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:35 smithi175 ceph-mon[84648]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:35.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:35 smithi175 ceph-mon[89733]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:28:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:36.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:36.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:37.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:37 smithi175 ceph-mon[84648]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:37.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:37 smithi175 ceph-mon[84648]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:37.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:37 smithi175 ceph-mon[89733]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:37.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:37 smithi175 ceph-mon[89733]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:37 smithi179 ceph-mon[67580]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:37 smithi179 ceph-mon[67580]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:38.484 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:28:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:38 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:38.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:38 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:38 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:39.241 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:28:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:39 smithi179 ceph-mon[67580]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:39 smithi175 ceph-mon[84648]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:39 smithi175 ceph-mon[89733]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:41.237 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": null, 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": false, 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "", 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": null, 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "", 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:28:41.238 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:28:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:41 smithi179 ceph-mon[67580]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:41.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:41 smithi175 ceph-mon[84648]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:41.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:41 smithi175 ceph-mon[89733]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:41.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:28:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:42 smithi179 ceph-mon[67580]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:42 smithi175 ceph-mon[84648]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:42 smithi175 ceph-mon[89733]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:43.556 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:43 smithi175 ceph-mon[84648]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:43.556 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:43 smithi175 ceph-mon[89733]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:43.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:43 smithi179 ceph-mon[67580]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:28:43] "GET /metrics HTTP/1.1" 200 38180 "" "Prometheus/2.43.0" 2024-05-25T06:28:43.964 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:28:43.964 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:28:43.965 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:28:44.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-05-25T06:28:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:44 smithi179 ceph-mon[67580]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:44 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1623164163' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:28:44.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:44 smithi175 ceph-mon[84648]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:44.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:44 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1623164163' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:28:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:44 smithi175 ceph-mon[89733]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:44 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1623164163' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:28:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:46 smithi179 ceph-mon[67580]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:47.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:46 smithi175 ceph-mon[84648]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:47.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:46 smithi175 ceph-mon[89733]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:28:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:47 smithi175 ceph-mon[84648]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:48.071 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:47 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:47 smithi175 ceph-mon[89733]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:48.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:47 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:47 smithi179 ceph-mon[67580]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:48.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:47 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:48.683 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:49.254 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:49.255 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:48 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:28:49.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:48 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:49.286 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-25T06:28:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:49 smithi179 ceph-mon[67580]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:28:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:49 smithi179 ceph-mon[67580]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-05-25T06:28:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:49 smithi179 ceph-mon[67580]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:49 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:50.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:49 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[84648]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[84648]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[84648]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[89733]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[89733]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[89733]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:28:50.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:49 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:28:51.211 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:50 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:51.211 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:50 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:51.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:50 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:28:51.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:28:51.273+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:51.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:51.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:51.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:51.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:51.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:51.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:51.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:51.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:51.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:51.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:51.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:51.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:51 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:51 smithi175 ceph-mon[89733]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:51 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:52.040 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.041 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:52.042 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:52.043 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:51 smithi175 ceph-mon[84648]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:51 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:52.044 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:52.045 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:28:52.046 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:52.047 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:51 smithi175 ceph-mon[84648]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:28:52.048 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:52.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:52.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:52.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.049 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.049 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:51 smithi175 ceph-mon[89733]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:28:52.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:51 smithi179 ceph-mon[67580]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-05-25T06:28:52.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:51 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:52.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:52.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:28:52.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:28:52.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:28:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:28:52.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:28:52.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:28:52.290 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:28:52.290 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:51 smithi179 ceph-mon[67580]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:28:52.549 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:28:53.236 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (6m) 38s ago 29m 28.6M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (6m) 4m ago 28m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 38s ago 28m 62.6M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (8m) 4m ago 33m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (18m) 38s ago 35m 511M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (4m) 38s ago 35m 53.5M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (5m) 4m ago 33m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (4m) 38s ago 33m 46.4M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (7m) 38s ago 29m 16.5M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (7m) 4m ago 29m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (2m) 38s ago 33m 52.4M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (109s) 38s ago 32m 50.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:28:53.237 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (43s) 38s ago 32m 12.2M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (32m) 38s ago 32m 56.1M 2992M 17.2.0 e1d6a67b021e 30e165112200 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (31m) 4m ago 31m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (31m) 4m ago 31m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (30m) 4m ago 30m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (30m) 4m ago 30m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (7m) 4m ago 29m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (28m) 38s ago 28m 78.1M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:28:53.238 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (28m) 4m ago 28m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-25T06:28:53.655 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 8 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:28:53.656 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:28:53.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:28:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:28:53] "GET /metrics HTTP/1.1" 200 38184 "" "Prometheus/2.43.0" 2024-05-25T06:28:54.003 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "3/8 daemons upgraded", 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading osd daemons", 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:28:54.004 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:28:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:28:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:28:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mgr 2024-05-25T06:28:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mon 2024-05-25T06:28:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-25T06:28:54.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:54 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/699908502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.809 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.809 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:54.809 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.809 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.809 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.809 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:28:54.810 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:28:54.810 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:54.810 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.810 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mgr 2024-05-25T06:28:54.810 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.810 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.811 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mon 2024-05-25T06:28:54.811 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.811 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.811 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.811 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-25T06:28:54.811 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/699908502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.812 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.812 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:54.812 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.812 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mgr 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.813 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.814 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mon 2024-05-25T06:28:54.814 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.814 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:54.814 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:54.814 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-25T06:28:54.814 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:54 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/699908502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:28:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:55 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all crash 2024-05-25T06:28:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:55 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-25T06:28:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:55 smithi179 ceph-mon[67580]: Upgrade: osd.3 is safe to restart 2024-05-25T06:28:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:55 smithi179 ceph-mon[67580]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:55.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:55 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:55.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:55 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T06:28:55.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:55 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:55.799 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all crash 2024-05-25T06:28:55.800 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-25T06:28:55.800 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[89733]: Upgrade: osd.3 is safe to restart 2024-05-25T06:28:55.800 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[89733]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:55.800 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:55.800 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T06:28:55.800 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:55.800 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all crash 2024-05-25T06:28:55.801 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-25T06:28:55.801 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[84648]: Upgrade: osd.3 is safe to restart 2024-05-25T06:28:55.801 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[84648]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:28:55.801 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:28:55.801 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-25T06:28:55.801 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:55 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:28:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:56 smithi179 ceph-mon[67580]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:56 smithi179 ceph-mon[67580]: Upgrade: Updating osd.3 2024-05-25T06:28:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:56 smithi179 ceph-mon[67580]: Deploying daemon osd.3 on smithi175 2024-05-25T06:28:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:56 smithi175 ceph-mon[84648]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:56 smithi175 ceph-mon[84648]: Upgrade: Updating osd.3 2024-05-25T06:28:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:56 smithi175 ceph-mon[84648]: Deploying daemon osd.3 on smithi175 2024-05-25T06:28:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:56 smithi175 ceph-mon[89733]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:56 smithi175 ceph-mon[89733]: Upgrade: Updating osd.3 2024-05-25T06:28:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:56 smithi175 ceph-mon[89733]: Deploying daemon osd.3 on smithi175 2024-05-25T06:28:58.705 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:58 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:58.706 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:58 smithi175 ceph-mon[84648]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:58.706 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:58 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:58.706 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:58 smithi175 ceph-mon[89733]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:58.706 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:28:58 smithi175 systemd[1]: Stopping Ceph osd.3 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:28:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:58 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:28:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:58 smithi179 ceph-mon[67580]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:28:59.070 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:28:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[41081]: 2024-05-25T06:28:58.705+0000 7ff3e7124700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:28:59.070 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:28:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[41081]: 2024-05-25T06:28:58.705+0000 7ff3e7124700 -1 osd.3 123 *** Got signal Terminated *** 2024-05-25T06:28:59.070 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:28:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[41081]: 2024-05-25T06:28:58.705+0000 7ff3e7124700 -1 osd.3 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:28:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:28:59 smithi179 ceph-mon[67580]: osd.3 marked itself down and dead 2024-05-25T06:28:59.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:28:59 smithi175 ceph-mon[84648]: osd.3 marked itself down and dead 2024-05-25T06:28:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:28:59 smithi175 ceph-mon[89733]: osd.3 marked itself down and dead 2024-05-25T06:29:00.213 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:28:59 smithi175 podman[116618]: 2024-05-25 06:28:59.941550232 +0000 UTC m=+1.364701662 container died 30e1651122008fb85447e2aa87ac66e75f79932a5ef8228cbb9a8a005dafd67c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, release=754) 2024-05-25T06:29:00.513 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:00 smithi175 podman[116618]: 2024-05-25 06:29:00.379562259 +0000 UTC m=+1.802713730 container cleanup 30e1651122008fb85447e2aa87ac66e75f79932a5ef8228cbb9a8a005dafd67c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, release=754, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container) 2024-05-25T06:29:00.513 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:00 smithi175 bash[116618]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3 2024-05-25T06:29:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:00 smithi179 ceph-mon[67580]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:29:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:00 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:00 smithi179 ceph-mon[67580]: osdmap e124: 8 total, 7 up, 8 in 2024-05-25T06:29:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:00 smithi175 ceph-mon[84648]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:29:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:00 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:00 smithi175 ceph-mon[84648]: osdmap e124: 8 total, 7 up, 8 in 2024-05-25T06:29:00.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:00 smithi175 ceph-mon[89733]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:29:00.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:00 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:00.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:00 smithi175 ceph-mon[89733]: osdmap e124: 8 total, 7 up, 8 in 2024-05-25T06:29:00.821 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:00 smithi175 podman[116633]: 2024-05-25 06:29:00.614744501 +0000 UTC m=+0.666283605 container remove 30e1651122008fb85447e2aa87ac66e75f79932a5ef8228cbb9a8a005dafd67c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vendor=Red Hat, Inc., release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-25T06:29:01.320 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:01 smithi175 podman[116721]: 2024-05-25 06:29:00.955853898 +0000 UTC m=+0.017291260 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:01.320 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:01 smithi175 podman[116721]: 2024-05-25 06:29:01.074181815 +0000 UTC m=+0.135619161 container create a11c02b42d75ab7e45583d065a2bc6722993be126628581498263cf72403b3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, RELEASE=squid-ea0684c, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-25T06:29:01.320 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:01 smithi175 podman[116721]: 2024-05-25 06:29:01.27664857 +0000 UTC m=+0.338085922 container init a11c02b42d75ab7e45583d065a2bc6722993be126628581498263cf72403b3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-05-25T06:29:01.320 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:01 smithi175 podman[116721]: 2024-05-25 06:29:01.279973023 +0000 UTC m=+0.341410379 container start a11c02b42d75ab7e45583d065a2bc6722993be126628581498263cf72403b3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-05-25T06:29:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:01 smithi179 ceph-mon[67580]: osdmap e125: 8 total, 7 up, 8 in 2024-05-25T06:29:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:01 smithi179 ceph-mon[67580]: pgmap v170: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:29:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:01 smithi175 ceph-mon[84648]: osdmap e125: 8 total, 7 up, 8 in 2024-05-25T06:29:01.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:01 smithi175 ceph-mon[84648]: pgmap v170: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:29:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:01 smithi175 ceph-mon[89733]: osdmap e125: 8 total, 7 up, 8 in 2024-05-25T06:29:01.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:01 smithi175 ceph-mon[89733]: pgmap v170: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:29:01.820 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:01 smithi175 podman[116721]: 2024-05-25 06:29:01.368521891 +0000 UTC m=+0.429959244 container attach a11c02b42d75ab7e45583d065a2bc6722993be126628581498263cf72403b3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, ceph=True) 2024-05-25T06:29:01.821 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:01 smithi175 podman[116721]: 2024-05-25 06:29:01.590312067 +0000 UTC m=+0.651749420 container died a11c02b42d75ab7e45583d065a2bc6722993be126628581498263cf72403b3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:29:02.570 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 podman[116763]: 2024-05-25 06:29:02.306031499 +0000 UTC m=+0.709678406 container remove a11c02b42d75ab7e45583d065a2bc6722993be126628581498263cf72403b3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, GIT_CLEAN=True) 2024-05-25T06:29:02.570 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.3.service: Deactivated successfully. 2024-05-25T06:29:02.570 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 systemd[1]: Stopped Ceph osd.3 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:29:02.570 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.3.service: Consumed 17.559s CPU time. 2024-05-25T06:29:02.570 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 systemd[1]: Starting Ceph osd.3 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:29:03.034 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:02 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 68/723 objects degraded (9.405%), 18 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:03.035 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:02 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 68/723 objects degraded (9.405%), 18 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:03.035 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 podman[116864]: 2024-05-25 06:29:02.723642334 +0000 UTC m=+0.097363743 container create 7fadc5530a03373489c5e542ac51fa065e5d59a704f26eb99db9b82f73623238 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-05-25T06:29:03.035 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 podman[116864]: 2024-05-25 06:29:02.644182705 +0000 UTC m=+0.017904120 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:03.035 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 podman[116864]: 2024-05-25 06:29:02.951670833 +0000 UTC m=+0.325392251 container init 7fadc5530a03373489c5e542ac51fa065e5d59a704f26eb99db9b82f73623238 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-05-25T06:29:03.035 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:02 smithi175 podman[116864]: 2024-05-25 06:29:02.955165202 +0000 UTC m=+0.328886612 container start 7fadc5530a03373489c5e542ac51fa065e5d59a704f26eb99db9b82f73623238 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:29:03.035 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 podman[116864]: 2024-05-25 06:29:03.034957245 +0000 UTC m=+0.408678665 container attach 7fadc5530a03373489c5e542ac51fa065e5d59a704f26eb99db9b82f73623238 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-25T06:29:03.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:02 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 68/723 objects degraded (9.405%), 18 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:03.771 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:29:03] "GET /metrics HTTP/1.1" 200 38184 "" "Prometheus/2.43.0" 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate[116878]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate[116878]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 bash[116864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 bash[116864]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 bash[116864]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 bash[116864]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 bash[116864]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 bash[116864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 bash[116864]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate[116878]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate[116878]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate[116878]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-25T06:29:03.772 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate[116878]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-25T06:29:03.773 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate[116878]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-25T06:29:03.773 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:03 smithi175 podman[116864]: 2024-05-25 06:29:03.432785855 +0000 UTC m=+0.806507262 container died 7fadc5530a03373489c5e542ac51fa065e5d59a704f26eb99db9b82f73623238 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-25T06:29:04.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:03 smithi175 ceph-mon[84648]: pgmap v171: 161 pgs: 18 active+undersized+degraded, 7 stale+active+clean, 28 active+undersized, 108 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 68/723 objects degraded (9.405%) 2024-05-25T06:29:04.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:03 smithi175 ceph-mon[89733]: pgmap v171: 161 pgs: 18 active+undersized+degraded, 7 stale+active+clean, 28 active+undersized, 108 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 68/723 objects degraded (9.405%) 2024-05-25T06:29:04.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:03 smithi179 ceph-mon[67580]: pgmap v171: 161 pgs: 18 active+undersized+degraded, 7 stale+active+clean, 28 active+undersized, 108 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 68/723 objects degraded (9.405%) 2024-05-25T06:29:04.383 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 podman[117011]: 2024-05-25 06:29:04.07274851 +0000 UTC m=+0.633912311 container remove 7fadc5530a03373489c5e542ac51fa065e5d59a704f26eb99db9b82f73623238 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:29:04.383 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 podman[117072]: 2024-05-25 06:29:04.282035295 +0000 UTC m=+0.111887469 container create 6e3e799cd1d8c3f219512bd1d88d2a52418d2df50d02f8050b5e389f08d8f2aa (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:29:04.383 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 podman[117072]: 2024-05-25 06:29:04.187503814 +0000 UTC m=+0.017356001 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:04.820 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 podman[117072]: 2024-05-25 06:29:04.509794342 +0000 UTC m=+0.339646531 container init 6e3e799cd1d8c3f219512bd1d88d2a52418d2df50d02f8050b5e389f08d8f2aa (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:29:04.820 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 podman[117072]: 2024-05-25 06:29:04.513108481 +0000 UTC m=+0.342960667 container start 6e3e799cd1d8c3f219512bd1d88d2a52418d2df50d02f8050b5e389f08d8f2aa (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-25T06:29:04.820 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 ceph-osd[117091]: -- 172.21.15.175:0/1018216359 <== mon.1 v2:172.21.15.175:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5582e9fed680 con 0x5582e9fb6c00 2024-05-25T06:29:04.820 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 bash[117072]: 6e3e799cd1d8c3f219512bd1d88d2a52418d2df50d02f8050b5e389f08d8f2aa 2024-05-25T06:29:04.820 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:04 smithi175 systemd[1]: Started Ceph osd.3 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:29:05.319 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:29:05.057+0000 7f79090167c0 -1 Falling back to public interface 2024-05-25T06:29:05.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[89733]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[84648]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:05.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:05 smithi179 ceph-mon[67580]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:07 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:08.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:07 smithi179 ceph-mon[67580]: pgmap v173: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:08.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:07 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:08.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:07 smithi175 ceph-mon[89733]: pgmap v173: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:07 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:07 smithi175 ceph-mon[84648]: pgmap v173: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:08.070 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:29:07.805+0000 7f79090167c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-05-25T06:29:08.070 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:07 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:29:07.839+0000 7f79090167c0 -1 osd.3 123 log_to_monitors true 2024-05-25T06:29:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:08 smithi179 ceph-mon[67580]: from='osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T06:29:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:08 smithi179 ceph-mon[67580]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T06:29:09.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:08 smithi175 ceph-mon[84648]: from='osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T06:29:09.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:08 smithi175 ceph-mon[84648]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T06:29:09.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:08 smithi175 ceph-mon[89733]: from='osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T06:29:09.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:08 smithi175 ceph-mon[89733]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-25T06:29:09.569 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:29:09 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:29:09.258+0000 7f78f6465700 -1 osd.3 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:29:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: pgmap v174: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-25T06:29:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: osdmap e126: 8 total, 7 up, 8 in 2024-05-25T06:29:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: from='osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:29:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:29:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:10.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:10.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:09 smithi179 ceph-mon[67580]: from='osd.3 ' entity='osd.3' 2024-05-25T06:29:10.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: pgmap v174: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:10.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: osdmap e126: 8 total, 7 up, 8 in 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: from='osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[84648]: from='osd.3 ' entity='osd.3' 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: pgmap v174: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: osdmap e126: 8 total, 7 up, 8 in 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: from='osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:10.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:10.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:09 smithi175 ceph-mon[89733]: from='osd.3 ' entity='osd.3' 2024-05-25T06:29:11.436 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:11.436 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:11.436 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:11.436 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[89733]: osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834] boot 2024-05-25T06:29:11.436 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[89733]: osdmap e127: 8 total, 8 up, 8 in 2024-05-25T06:29:11.436 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:29:11.437 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:11.437 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:11.437 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:11.437 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[84648]: osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834] boot 2024-05-25T06:29:11.437 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[84648]: osdmap e127: 8 total, 8 up, 8 in 2024-05-25T06:29:11.437 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:29:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:11 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:11 smithi179 ceph-mon[67580]: osd.3 [v2:172.21.15.175:6826/394233834,v1:172.21.15.175:6827/394233834] boot 2024-05-25T06:29:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:11 smithi179 ceph-mon[67580]: osdmap e127: 8 total, 8 up, 8 in 2024-05-25T06:29:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-25T06:29:12.258 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:12 smithi175 ceph-mon[84648]: pgmap v177: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:29:12.258 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:12 smithi175 ceph-mon[84648]: osdmap e128: 8 total, 8 up, 8 in 2024-05-25T06:29:12.258 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:12 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 26 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:12 smithi179 ceph-mon[67580]: pgmap v177: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:29:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:12 smithi179 ceph-mon[67580]: osdmap e128: 8 total, 8 up, 8 in 2024-05-25T06:29:12.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:12 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 26 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:12.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:12 smithi175 ceph-mon[89733]: pgmap v177: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:29:12.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:12 smithi175 ceph-mon[89733]: osdmap e128: 8 total, 8 up, 8 in 2024-05-25T06:29:12.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:12 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 26 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:13.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:29:13] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-05-25T06:29:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: pgmap v179: 161 pgs: 15 active+undersized+degraded, 25 active+undersized, 121 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-05-25T06:29:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:14.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:14.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:14.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:14 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:14.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: pgmap v179: 161 pgs: 15 active+undersized+degraded, 25 active+undersized, 121 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-05-25T06:29:14.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:14.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:14.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: pgmap v179: 161 pgs: 15 active+undersized+degraded, 25 active+undersized, 121 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-05-25T06:29:14.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:14.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:14.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:14.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:14.573 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:14 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[84648]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[84648]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 15 pgs degraded) 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[89733]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[89733]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:15.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:15 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 15 pgs degraded) 2024-05-25T06:29:15.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:15 smithi179 ceph-mon[67580]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:15.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:15 smithi179 ceph-mon[67580]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:15.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:15 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:15.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:15 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:15.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:15 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 15 pgs degraded) 2024-05-25T06:29:16.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:29:15.864+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:16.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:29:16.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:16.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:16.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:16.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:16.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:16.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:16.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:16.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:16.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:16.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:16.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:16.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:16.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:15 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-25T06:29:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-25T06:29:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:16.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-25T06:29:17.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:17 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:17.535 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:17.535 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:17.535 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:17.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:17.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:17.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:17.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.541 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:17 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-25T06:29:17.541 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:17 smithi179 ceph-mon[67580]: Upgrade: osd.4 is safe to restart 2024-05-25T06:29:17.541 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:17 smithi179 ceph-mon[67580]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:29:17.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:17.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:17.574 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:17.574 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:17.574 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:17.574 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:17.574 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:17.575 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:17.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:17.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:17.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:17.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:17.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:17.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:17.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[89733]: Upgrade: osd.4 is safe to restart 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[89733]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:17.580 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:17.581 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:17.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-25T06:29:17.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[84648]: Upgrade: osd.4 is safe to restart 2024-05-25T06:29:17.581 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:17 smithi175 ceph-mon[84648]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:18 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:18 smithi179 ceph-mon[67580]: Upgrade: Updating osd.4 2024-05-25T06:29:18.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:18.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T06:29:18.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:18.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:18 smithi179 ceph-mon[67580]: Deploying daemon osd.4 on smithi179 2024-05-25T06:29:18.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:18.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[84648]: Upgrade: Updating osd.4 2024-05-25T06:29:18.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[84648]: Deploying daemon osd.4 on smithi179 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[89733]: Upgrade: Updating osd.4 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:18 smithi175 ceph-mon[89733]: Deploying daemon osd.4 on smithi179 2024-05-25T06:29:19.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:19 smithi179 ceph-mon[67580]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:19.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:19 smithi175 ceph-mon[84648]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:19.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:19 smithi175 ceph-mon[89733]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:20.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:20 smithi179 systemd[1]: Stopping Ceph osd.4 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:29:21.036 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[28968]: 2024-05-25T06:29:20.707+0000 7fef63e86700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:29:21.036 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[28968]: 2024-05-25T06:29:20.707+0000 7fef63e86700 -1 osd.4 128 *** Got signal Terminated *** 2024-05-25T06:29:21.036 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[28968]: 2024-05-25T06:29:20.707+0000 7fef63e86700 -1 osd.4 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:29:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:21 smithi179 ceph-mon[67580]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 235 B/s rd, 0 op/s 2024-05-25T06:29:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:21 smithi179 ceph-mon[67580]: osd.4 marked itself down and dead 2024-05-25T06:29:21.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[84648]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 235 B/s rd, 0 op/s 2024-05-25T06:29:21.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[84648]: osd.4 marked itself down and dead 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[89733]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 235 B/s rd, 0 op/s 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:21.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:21 smithi175 ceph-mon[89733]: osd.4 marked itself down and dead 2024-05-25T06:29:22.184 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:21 smithi179 podman[77317]: 2024-05-25 06:29:21.9098288 +0000 UTC m=+1.311140857 container died 5f44d6596b9f74336d1fd817a44e26fdb0c283a93a323874a9e3c3adef3ca734 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-type=git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, architecture=x86_64, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, RELEASE=HEAD) 2024-05-25T06:29:22.497 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:22 smithi179 podman[77317]: 2024-05-25 06:29:22.291714004 +0000 UTC m=+1.693026036 container cleanup 5f44d6596b9f74336d1fd817a44e26fdb0c283a93a323874a9e3c3adef3ca734 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, name=centos-stream, io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, RELEASE=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8) 2024-05-25T06:29:22.498 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:22 smithi179 bash[77317]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4 2024-05-25T06:29:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:22 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:22 smithi179 ceph-mon[67580]: osdmap e129: 8 total, 7 up, 8 in 2024-05-25T06:29:22.785 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:22 smithi179 podman[77345]: 2024-05-25 06:29:22.5349984 +0000 UTC m=+0.619853512 container remove 5f44d6596b9f74336d1fd817a44e26fdb0c283a93a323874a9e3c3adef3ca734 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-type=git, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, RELEASE=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0) 2024-05-25T06:29:22.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:22 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:22.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:22 smithi175 ceph-mon[84648]: osdmap e129: 8 total, 7 up, 8 in 2024-05-25T06:29:22.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:22 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:22.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:22 smithi175 ceph-mon[89733]: osdmap e129: 8 total, 7 up, 8 in 2024-05-25T06:29:23.163 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:22 smithi179 podman[77432]: 2024-05-25 06:29:22.779782966 +0000 UTC m=+0.017670210 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:23.163 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:22 smithi179 podman[77432]: 2024-05-25 06:29:22.877992334 +0000 UTC m=+0.115879574 container create 7cb291377de60ad0865167865c5e3b0a35616b23ea7779a834106f665e6be679 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-25T06:29:23.163 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:23 smithi179 podman[77432]: 2024-05-25 06:29:23.072248256 +0000 UTC m=+0.310135498 container init 7cb291377de60ad0865167865c5e3b0a35616b23ea7779a834106f665e6be679 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240521) 2024-05-25T06:29:23.163 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:23 smithi179 podman[77432]: 2024-05-25 06:29:23.075456186 +0000 UTC m=+0.313343428 container start 7cb291377de60ad0865167865c5e3b0a35616b23ea7779a834106f665e6be679 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:29:23.163 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:23 smithi179 podman[77432]: 2024-05-25 06:29:23.16381842 +0000 UTC m=+0.401705679 container attach 7cb291377de60ad0865167865c5e3b0a35616b23ea7779a834106f665e6be679 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-25T06:29:23.498 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:23 smithi179 podman[77432]: 2024-05-25 06:29:23.374205389 +0000 UTC m=+0.612092631 container died 7cb291377de60ad0865167865c5e3b0a35616b23ea7779a834106f665e6be679 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:29:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:23 smithi179 ceph-mon[67580]: pgmap v186: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:29:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:23 smithi179 ceph-mon[67580]: osdmap e130: 8 total, 7 up, 8 in 2024-05-25T06:29:23.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:23 smithi175 ceph-mon[84648]: pgmap v186: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:29:23.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:23 smithi175 ceph-mon[84648]: osdmap e130: 8 total, 7 up, 8 in 2024-05-25T06:29:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:23 smithi175 ceph-mon[89733]: pgmap v186: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-25T06:29:23.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:23 smithi175 ceph-mon[89733]: osdmap e130: 8 total, 7 up, 8 in 2024-05-25T06:29:23.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:29:23] "GET /metrics HTTP/1.1" 200 38190 "" "Prometheus/2.43.0" 2024-05-25T06:29:24.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:23 smithi179 podman[77457]: 2024-05-25 06:29:23.992841792 +0000 UTC m=+0.612746063 container remove 7cb291377de60ad0865167865c5e3b0a35616b23ea7779a834106f665e6be679 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-25T06:29:24.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.4.service: Deactivated successfully. 2024-05-25T06:29:24.286 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 systemd[1]: Stopped Ceph osd.4 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:29:24.286 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.4.service: Consumed 17.006s CPU time. 2024-05-25T06:29:24.286 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 systemd[1]: Starting Ceph osd.4 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:29:24.368 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:29:24.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:24 smithi179 ceph-mon[67580]: pgmap v188: 161 pgs: 8 active+undersized+degraded, 11 stale+active+clean, 19 active+undersized, 123 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-05-25T06:29:24.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:24 smithi179 ceph-mon[67580]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:24.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:24 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:24.721 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 podman[77557]: 2024-05-25 06:29:24.4021988 +0000 UTC m=+0.092824565 container create 2d4d6cd29da19c588c1a5b7bca599a1504381347ec047f8a2d611cfe87ed648e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-05-25T06:29:24.721 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 podman[77557]: 2024-05-25 06:29:24.326970266 +0000 UTC m=+0.017596051 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:24.721 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 podman[77557]: 2024-05-25 06:29:24.646818529 +0000 UTC m=+0.337444298 container init 2d4d6cd29da19c588c1a5b7bca599a1504381347ec047f8a2d611cfe87ed648e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-05-25T06:29:24.721 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 podman[77557]: 2024-05-25 06:29:24.650351164 +0000 UTC m=+0.340976929 container start 2d4d6cd29da19c588c1a5b7bca599a1504381347ec047f8a2d611cfe87ed648e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, ceph=True, RELEASE=squid-ea0684c) 2024-05-25T06:29:24.722 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:24 smithi179 podman[77557]: 2024-05-25 06:29:24.721613483 +0000 UTC m=+0.412239248 container attach 2d4d6cd29da19c588c1a5b7bca599a1504381347ec047f8a2d611cfe87ed648e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True) 2024-05-25T06:29:24.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:24 smithi175 ceph-mon[89733]: pgmap v188: 161 pgs: 8 active+undersized+degraded, 11 stale+active+clean, 19 active+undersized, 123 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-05-25T06:29:24.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:24 smithi175 ceph-mon[89733]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:24.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:24 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:24 smithi175 ceph-mon[84648]: pgmap v188: 161 pgs: 8 active+undersized+degraded, 11 stale+active+clean, 19 active+undersized, 123 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-05-25T06:29:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:24 smithi175 ceph-mon[84648]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:24.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:24 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:25.026 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (7m) 16s ago 29m 28.6M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (7m) 4m ago 29m 79.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 16s ago 28m 62.7M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (8m) 4m ago 34m 449M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (18m) 16s ago 35m 513M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (4m) 16s ago 35m 53.4M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (5m) 4m ago 34m 36.7M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (4m) 16s ago 34m 44.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (8m) 16s ago 30m 16.1M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (8m) 4m ago 30m 14.9M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (2m) 16s ago 33m 53.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (2m) 16s ago 33m 51.4M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (74s) 16s ago 33m 41.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:29:25.027 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (20s) 16s ago 32m 14.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:29:25.028 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (32m) 4m ago 32m 56.1M 3248M 17.2.0 e1d6a67b021e 5f44d6596b9f 2024-05-25T06:29:25.028 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (31m) 4m ago 31m 52.8M 3248M 17.2.0 e1d6a67b021e 20a5caafa0ae 2024-05-25T06:29:25.028 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (31m) 4m ago 31m 55.2M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:29:25.028 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (31m) 4m ago 31m 57.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:29:25.028 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (7m) 4m ago 30m 68.5M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:29:25.028 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (28m) 16s ago 28m 78.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:29:25.028 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (28m) 4m ago 28m 79.9M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:29:25.421 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 4 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 9 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:29:25.422 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:29:25.424 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate[77571]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-25T06:29:25.424 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate[77571]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-25T06:29:25.424 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 bash[77557]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 bash[77557]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 bash[77557]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 bash[77557]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 bash[77557]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 bash[77557]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 bash[77557]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate[77571]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate[77571]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate[77571]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate[77571]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate[77571]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-25T06:29:25.425 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 podman[77557]: 2024-05-25 06:29:25.12087782 +0000 UTC m=+0.811503585 container died 2d4d6cd29da19c588c1a5b7bca599a1504381347ec047f8a2d611cfe87ed648e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD) 2024-05-25T06:29:25.725 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:25 smithi179 ceph-mon[67580]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:25.725 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:25 smithi179 ceph-mon[67580]: from='client.54421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:25.725 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:25 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2616027554' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:25.754 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:29:25.754 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:29:25.755 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:29:25.755 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-25T06:29:25.755 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:29:25.755 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "4/8 daemons upgraded", 2024-05-25T06:29:25.755 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading osd daemons", 2024-05-25T06:29:25.755 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:29:25.755 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:29:25.755 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:25 smithi175 ceph-mon[89733]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:25.755 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:25 smithi175 ceph-mon[89733]: from='client.54421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:25.756 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:25 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2616027554' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:25.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:25 smithi175 ceph-mon[84648]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:25.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:25 smithi175 ceph-mon[84648]: from='client.54421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:25.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:25 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2616027554' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:26.035 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 podman[77692]: 2024-05-25 06:29:25.725708148 +0000 UTC m=+0.597967477 container remove 2d4d6cd29da19c588c1a5b7bca599a1504381347ec047f8a2d611cfe87ed648e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-25T06:29:26.035 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 podman[77754]: 2024-05-25 06:29:25.842503787 +0000 UTC m=+0.017619580 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:26.035 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:25 smithi179 podman[77754]: 2024-05-25 06:29:25.960111562 +0000 UTC m=+0.135227328 container create cac9395d4e632609b549e45d73373db5400c30e3782092c1b473daf7088e7c41 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-25T06:29:26.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:26 smithi179 podman[77754]: 2024-05-25 06:29:26.146261717 +0000 UTC m=+0.321377497 container init cac9395d4e632609b549e45d73373db5400c30e3782092c1b473daf7088e7c41 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:29:26.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:26 smithi179 podman[77754]: 2024-05-25 06:29:26.14936856 +0000 UTC m=+0.324484327 container start cac9395d4e632609b549e45d73373db5400c30e3782092c1b473daf7088e7c41 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-05-25T06:29:26.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:26 smithi179 ceph-osd[77772]: -- 172.21.15.179:0/854406855 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a5dbceb680 con 0x55a5dbcb5400 2024-05-25T06:29:26.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:26 smithi179 bash[77754]: cac9395d4e632609b549e45d73373db5400c30e3782092c1b473daf7088e7c41 2024-05-25T06:29:26.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:26 smithi179 systemd[1]: Started Ceph osd.4 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:29:27.035 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:29:26.695+0000 7fa2c2f877c0 -1 Falling back to public interface 2024-05-25T06:29:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:27 smithi179 ceph-mon[67580]: from='client.54433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:27 smithi179 ceph-mon[67580]: pgmap v189: 161 pgs: 8 active+undersized+degraded, 11 stale+active+clean, 19 active+undersized, 123 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-05-25T06:29:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:27.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[84648]: from='client.54433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:27.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[84648]: pgmap v189: 161 pgs: 8 active+undersized+degraded, 11 stale+active+clean, 19 active+undersized, 123 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-05-25T06:29:27.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:27.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:27.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:27.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[89733]: from='client.54433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:27.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[89733]: pgmap v189: 161 pgs: 8 active+undersized+degraded, 11 stale+active+clean, 19 active+undersized, 123 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-05-25T06:29:27.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:27.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:27.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:28.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:28 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:28.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:28 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:28.654 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:28 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:29 smithi179 ceph-mon[67580]: pgmap v190: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 127/723 objects degraded (17.566%) 2024-05-25T06:29:29.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:29:29.401+0000 7fa2c2f877c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-05-25T06:29:29.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:29:29.433+0000 7fa2c2f877c0 -1 osd.4 128 log_to_monitors true 2024-05-25T06:29:29.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:29 smithi175 ceph-mon[84648]: pgmap v190: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 127/723 objects degraded (17.566%) 2024-05-25T06:29:29.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:29 smithi175 ceph-mon[89733]: pgmap v190: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 127/723 objects degraded (17.566%) 2024-05-25T06:29:30.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:30 smithi175 ceph-mon[89733]: from='osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T06:29:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:30 smithi175 ceph-mon[89733]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T06:29:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:30 smithi175 ceph-mon[84648]: from='osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T06:29:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:30 smithi175 ceph-mon[84648]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T06:29:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:30 smithi179 ceph-mon[67580]: from='osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T06:29:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:30 smithi179 ceph-mon[67580]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-25T06:29:30.785 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:29:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:29:30.323+0000 7fa2b43de700 -1 osd.4 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:29:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:31 smithi179 ceph-mon[67580]: pgmap v191: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 127/723 objects degraded (17.566%) 2024-05-25T06:29:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:31 smithi179 ceph-mon[67580]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-25T06:29:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:31 smithi179 ceph-mon[67580]: osdmap e131: 8 total, 7 up, 8 in 2024-05-25T06:29:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:31 smithi179 ceph-mon[67580]: from='osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:31 smithi179 ceph-mon[67580]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[84648]: pgmap v191: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 127/723 objects degraded (17.566%) 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[84648]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[84648]: osdmap e131: 8 total, 7 up, 8 in 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[84648]: from='osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[84648]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[89733]: pgmap v191: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 178 MiB used, 715 GiB / 715 GiB avail; 127/723 objects degraded (17.566%) 2024-05-25T06:29:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[89733]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-25T06:29:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[89733]: osdmap e131: 8 total, 7 up, 8 in 2024-05-25T06:29:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[89733]: from='osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[89733]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:31.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:32.450 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:32.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769] boot 2024-05-25T06:29:32.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: osdmap e132: 8 total, 8 up, 8 in 2024-05-25T06:29:32.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:29:32.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:32.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:32.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 127/723 objects degraded (17.566%), 28 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:32.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:32 smithi179 ceph-mon[67580]: osdmap e133: 8 total, 8 up, 8 in 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769] boot 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: osdmap e132: 8 total, 8 up, 8 in 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 127/723 objects degraded (17.566%), 28 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[89733]: osdmap e133: 8 total, 8 up, 8 in 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: osd.4 [v2:172.21.15.179:6800/911151769,v1:172.21.15.179:6801/911151769] boot 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: osdmap e132: 8 total, 8 up, 8 in 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 127/723 objects degraded (17.566%), 28 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:32.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:32 smithi175 ceph-mon[84648]: osdmap e133: 8 total, 8 up, 8 in 2024-05-25T06:29:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:33 smithi179 ceph-mon[67580]: pgmap v194: 161 pgs: 30 peering, 13 active+undersized+degraded, 25 active+undersized, 93 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 52/723 objects degraded (7.192%) 2024-05-25T06:29:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:33 smithi175 ceph-mon[84648]: pgmap v194: 161 pgs: 30 peering, 13 active+undersized+degraded, 25 active+undersized, 93 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 52/723 objects degraded (7.192%) 2024-05-25T06:29:33.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:33 smithi175 ceph-mon[89733]: pgmap v194: 161 pgs: 30 peering, 13 active+undersized+degraded, 25 active+undersized, 93 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 52/723 objects degraded (7.192%) 2024-05-25T06:29:33.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:29:33] "GET /metrics HTTP/1.1" 200 38190 "" "Prometheus/2.43.0" 2024-05-25T06:29:35.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: pgmap v196: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:35.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:35 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: pgmap v196: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: pgmap v196: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:35.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:35 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:36 smithi179 ceph-mon[67580]: pgmap v197: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:36 smithi179 ceph-mon[67580]: pgmap v198: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:36 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:36 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:36.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[89733]: pgmap v197: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[89733]: pgmap v198: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[84648]: pgmap v197: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[84648]: pgmap v198: 161 pgs: 30 peering, 10 active+undersized+degraded, 13 active+undersized, 108 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-05-25T06:29:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:36 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:37.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:29:36.960+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:37.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:29:37.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-25T06:29:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:29:37.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:37.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-25T06:29:37.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-25T06:29:37.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: Upgrade: osd.5 is safe to restart 2024-05-25T06:29:37.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:37 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 35/723 objects degraded (4.841%), 10 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:37.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:37.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:37.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:37.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:37.827 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: Upgrade: osd.5 is safe to restart 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 35/723 objects degraded (4.841%), 10 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:37.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-25T06:29:37.829 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: Upgrade: osd.5 is safe to restart 2024-05-25T06:29:37.830 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:37 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 35/723 objects degraded (4.841%), 10 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:38 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/723 objects degraded (4.841%), 10 pgs degraded) 2024-05-25T06:29:38.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:38 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/723 objects degraded (4.841%), 10 pgs degraded) 2024-05-25T06:29:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:38 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/723 objects degraded (4.841%), 10 pgs degraded) 2024-05-25T06:29:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:39 smithi179 ceph-mon[67580]: Upgrade: Updating osd.5 2024-05-25T06:29:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T06:29:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:39 smithi179 ceph-mon[67580]: Deploying daemon osd.5 on smithi179 2024-05-25T06:29:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:39 smithi179 ceph-mon[67580]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-05-25T06:29:39.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[84648]: Upgrade: Updating osd.5 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[84648]: Deploying daemon osd.5 on smithi179 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[84648]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[89733]: Upgrade: Updating osd.5 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-25T06:29:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[89733]: Deploying daemon osd.5 on smithi179 2024-05-25T06:29:39.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:39 smithi175 ceph-mon[89733]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-05-25T06:29:42.107 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:41 smithi179 ceph-mon[67580]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:29:42.107 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:41 smithi179 systemd[1]: Stopping Ceph osd.5 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:29:42.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:41 smithi175 ceph-mon[84648]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:29:42.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:41 smithi175 ceph-mon[89733]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:29:42.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:42 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[31907]: 2024-05-25T06:29:42.105+0000 7ff125fa6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:29:42.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:42 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[31907]: 2024-05-25T06:29:42.105+0000 7ff125fa6700 -1 osd.5 133 *** Got signal Terminated *** 2024-05-25T06:29:42.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:42 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[31907]: 2024-05-25T06:29:42.105+0000 7ff125fa6700 -1 osd.5 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:29:43.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:43 smithi175 ceph-mon[84648]: osd.5 marked itself down and dead 2024-05-25T06:29:43.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:43 smithi175 ceph-mon[89733]: osd.5 marked itself down and dead 2024-05-25T06:29:43.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:43 smithi179 ceph-mon[67580]: osd.5 marked itself down and dead 2024-05-25T06:29:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:29:43] "GET /metrics HTTP/1.1" 200 38202 "" "Prometheus/2.43.0" 2024-05-25T06:29:43.865 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:43 smithi179 podman[81790]: 2024-05-25 06:29:43.607210509 +0000 UTC m=+1.618029836 container died 20a5caafa0ae20d0694f72e8dc08fefc04f92afb80746f381e7e4a0031fbe83d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_BRANCH=HEAD, ceph=True, architecture=x86_64, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, name=centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-25T06:29:44.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:44 smithi179 ceph-mon[67580]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 575 B/s rd, 0 op/s 2024-05-25T06:29:44.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:44 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:44.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:44 smithi179 ceph-mon[67580]: osdmap e134: 8 total, 7 up, 8 in 2024-05-25T06:29:44.258 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:44 smithi179 podman[81790]: 2024-05-25 06:29:44.022873553 +0000 UTC m=+2.033692867 container cleanup 20a5caafa0ae20d0694f72e8dc08fefc04f92afb80746f381e7e4a0031fbe83d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-25T06:29:44.258 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:44 smithi179 bash[81790]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5 2024-05-25T06:29:44.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:44 smithi175 ceph-mon[89733]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 575 B/s rd, 0 op/s 2024-05-25T06:29:44.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:44 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:44 smithi175 ceph-mon[89733]: osdmap e134: 8 total, 7 up, 8 in 2024-05-25T06:29:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:44 smithi175 ceph-mon[84648]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 575 B/s rd, 0 op/s 2024-05-25T06:29:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:44 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:29:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:44 smithi175 ceph-mon[84648]: osdmap e134: 8 total, 7 up, 8 in 2024-05-25T06:29:44.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:44 smithi179 podman[81805]: 2024-05-25 06:29:44.257837384 +0000 UTC m=+0.648642126 container remove 20a5caafa0ae20d0694f72e8dc08fefc04f92afb80746f381e7e4a0031fbe83d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.component=centos-stream-container, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-25T06:29:44.994 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:44 smithi179 podman[81896]: 2024-05-25 06:29:44.619498018 +0000 UTC m=+0.017279807 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:44.995 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:44 smithi179 podman[81896]: 2024-05-25 06:29:44.73395276 +0000 UTC m=+0.131734530 container create 171cc267ba64a751e9412e717dafb220de6b7f1d9663db0fd6f76fb13fe0e3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-ea0684c) 2024-05-25T06:29:44.995 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:44 smithi179 podman[81896]: 2024-05-25 06:29:44.995403319 +0000 UTC m=+0.393185104 container init 171cc267ba64a751e9412e717dafb220de6b7f1d9663db0fd6f76fb13fe0e3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c) 2024-05-25T06:29:45.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:45 smithi179 ceph-mon[67580]: osdmap e135: 8 total, 7 up, 8 in 2024-05-25T06:29:45.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:44 smithi179 podman[81896]: 2024-05-25 06:29:44.999498466 +0000 UTC m=+0.397280242 container start 171cc267ba64a751e9412e717dafb220de6b7f1d9663db0fd6f76fb13fe0e3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, ceph=True, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:29:45.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:45 smithi179 podman[81896]: 2024-05-25 06:29:45.069946265 +0000 UTC m=+0.467728056 container attach 171cc267ba64a751e9412e717dafb220de6b7f1d9663db0fd6f76fb13fe0e3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c) 2024-05-25T06:29:45.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:45 smithi175 ceph-mon[89733]: osdmap e135: 8 total, 7 up, 8 in 2024-05-25T06:29:45.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:45 smithi175 ceph-mon[84648]: osdmap e135: 8 total, 7 up, 8 in 2024-05-25T06:29:45.581 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:45 smithi179 podman[81896]: 2024-05-25 06:29:45.29137321 +0000 UTC m=+0.689155001 container died 171cc267ba64a751e9412e717dafb220de6b7f1d9663db0fd6f76fb13fe0e3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, ceph=True, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521) 2024-05-25T06:29:46.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:46 smithi179 ceph-mon[67580]: pgmap v205: 161 pgs: 28 peering, 10 stale+active+clean, 123 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:46.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:46 smithi179 ceph-mon[67580]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-05-25T06:29:46.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:46 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:46.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:45 smithi179 podman[81922]: 2024-05-25 06:29:45.857035305 +0000 UTC m=+0.560105613 container remove 171cc267ba64a751e9412e717dafb220de6b7f1d9663db0fd6f76fb13fe0e3fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-25T06:29:46.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:45 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.5.service: Deactivated successfully. 2024-05-25T06:29:46.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:45 smithi179 systemd[1]: Stopped Ceph osd.5 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:29:46.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:45 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.5.service: Consumed 16.741s CPU time. 2024-05-25T06:29:46.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:46 smithi179 systemd[1]: Starting Ceph osd.5 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:29:46.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:46 smithi175 ceph-mon[84648]: pgmap v205: 161 pgs: 28 peering, 10 stale+active+clean, 123 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:46.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:46 smithi175 ceph-mon[84648]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-05-25T06:29:46.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:46 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:46.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:46 smithi175 ceph-mon[89733]: pgmap v205: 161 pgs: 28 peering, 10 stale+active+clean, 123 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:29:46.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:46 smithi175 ceph-mon[89733]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-05-25T06:29:46.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:46 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:46.561 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:46 smithi179 podman[82021]: 2024-05-25 06:29:46.191107093 +0000 UTC m=+0.017545956 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:46.561 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:46 smithi179 podman[82021]: 2024-05-25 06:29:46.31667544 +0000 UTC m=+0.143114283 container create 84ff5711bd7c545c3d4550fcb58891ad16dfefed2a3d7c2af2be1503e41f1f73 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:29:46.561 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:46 smithi179 podman[82021]: 2024-05-25 06:29:46.561844531 +0000 UTC m=+0.388283390 container init 84ff5711bd7c545c3d4550fcb58891ad16dfefed2a3d7c2af2be1503e41f1f73 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:29:47.008 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:46 smithi179 podman[82021]: 2024-05-25 06:29:46.565434118 +0000 UTC m=+0.391872969 container start 84ff5711bd7c545c3d4550fcb58891ad16dfefed2a3d7c2af2be1503e41f1f73 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240521) 2024-05-25T06:29:47.009 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:46 smithi179 podman[82021]: 2024-05-25 06:29:46.644772864 +0000 UTC m=+0.471211709 container attach 84ff5711bd7c545c3d4550fcb58891ad16dfefed2a3d7c2af2be1503e41f1f73 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:29:47.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:47 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 47/723 objects degraded (6.501%), 14 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:47.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate[82037]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-25T06:29:47.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate[82037]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 bash[82021]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 bash[82021]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 bash[82021]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 bash[82021]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 bash[82021]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 bash[82021]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 bash[82021]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate[82037]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate[82037]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate[82037]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate[82037]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate[82037]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-25T06:29:47.286 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 podman[82021]: 2024-05-25 06:29:47.033796225 +0000 UTC m=+0.860235073 container died 84ff5711bd7c545c3d4550fcb58891ad16dfefed2a3d7c2af2be1503e41f1f73 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-05-25T06:29:47.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:47 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 47/723 objects degraded (6.501%), 14 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:47.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:47 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 47/723 objects degraded (6.501%), 14 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:47.890 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 podman[82153]: 2024-05-25 06:29:47.631967377 +0000 UTC m=+0.592210102 container remove 84ff5711bd7c545c3d4550fcb58891ad16dfefed2a3d7c2af2be1503e41f1f73 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, ceph=True) 2024-05-25T06:29:47.891 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 podman[82213]: 2024-05-25 06:29:47.749339796 +0000 UTC m=+0.017485314 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:29:47.891 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:47 smithi179 podman[82213]: 2024-05-25 06:29:47.891509057 +0000 UTC m=+0.159654559 container create 5a1d82fca8df5799cc88ee63b0d7e01214343bc711da617f4ee0cd4ea52f3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:29:48.160 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:48 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:48.160 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:48 smithi179 ceph-mon[67580]: pgmap v206: 161 pgs: 14 active+undersized+degraded, 28 peering, 17 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 47/723 objects degraded (6.501%) 2024-05-25T06:29:48.161 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:48 smithi179 podman[82213]: 2024-05-25 06:29:48.136247206 +0000 UTC m=+0.404392717 container init 5a1d82fca8df5799cc88ee63b0d7e01214343bc711da617f4ee0cd4ea52f3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, GIT_CLEAN=True, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-05-25T06:29:48.161 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:48 smithi179 podman[82213]: 2024-05-25 06:29:48.140510612 +0000 UTC m=+0.408656124 container start 5a1d82fca8df5799cc88ee63b0d7e01214343bc711da617f4ee0cd4ea52f3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, RELEASE=squid-ea0684c, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-05-25T06:29:48.465 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:48 smithi179 ceph-osd[82234]: -- 172.21.15.179:0/1654853134 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5632b10c7680 con 0x5632b1091000 2024-05-25T06:29:48.465 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:48 smithi179 bash[82213]: 5a1d82fca8df5799cc88ee63b0d7e01214343bc711da617f4ee0cd4ea52f3176 2024-05-25T06:29:48.465 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:48 smithi179 systemd[1]: Started Ceph osd.5 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:29:48.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:48 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:48.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:48 smithi175 ceph-mon[84648]: pgmap v206: 161 pgs: 14 active+undersized+degraded, 28 peering, 17 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 47/723 objects degraded (6.501%) 2024-05-25T06:29:48.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:48 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:48.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:48 smithi175 ceph-mon[89733]: pgmap v206: 161 pgs: 14 active+undersized+degraded, 28 peering, 17 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 47/723 objects degraded (6.501%) 2024-05-25T06:29:48.785 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:48 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:29:48.685+0000 7faf9eae27c0 -1 Falling back to public interface 2024-05-25T06:29:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:49.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:49.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:49.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:49.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:49.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:50 smithi175 ceph-mon[84648]: pgmap v207: 161 pgs: 15 active+undersized+degraded, 10 peering, 34 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-05-25T06:29:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:50 smithi175 ceph-mon[89733]: pgmap v207: 161 pgs: 15 active+undersized+degraded, 10 peering, 34 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-05-25T06:29:50.702 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:50 smithi179 ceph-mon[67580]: pgmap v207: 161 pgs: 15 active+undersized+degraded, 10 peering, 34 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-05-25T06:29:51.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:51.717 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:51.717 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:51 smithi179 ceph-mon[67580]: from='osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T06:29:51.717 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:51 smithi179 ceph-mon[67580]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T06:29:51.717 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:29:51.394+0000 7faf9eae27c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-05-25T06:29:51.717 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:29:51.419+0000 7faf9eae27c0 -1 osd.5 133 log_to_monitors true 2024-05-25T06:29:51.717 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:29:51 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:29:51.523+0000 7faf8ff39700 -1 osd.5 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[84648]: from='osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[84648]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[89733]: from='osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T06:29:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:51 smithi175 ceph-mon[89733]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-25T06:29:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:52 smithi179 ceph-mon[67580]: pgmap v208: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 78/723 objects degraded (10.788%) 2024-05-25T06:29:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:52 smithi179 ceph-mon[67580]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-05-25T06:29:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:52 smithi179 ceph-mon[67580]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-25T06:29:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:52 smithi179 ceph-mon[67580]: osdmap e136: 8 total, 7 up, 8 in 2024-05-25T06:29:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:52 smithi179 ceph-mon[67580]: from='osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:52 smithi179 ceph-mon[67580]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:52.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[84648]: pgmap v208: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 78/723 objects degraded (10.788%) 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[84648]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[84648]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[84648]: osdmap e136: 8 total, 7 up, 8 in 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[84648]: from='osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[84648]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[89733]: pgmap v208: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 78/723 objects degraded (10.788%) 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[89733]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[89733]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[89733]: osdmap e136: 8 total, 7 up, 8 in 2024-05-25T06:29:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[89733]: from='osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:52.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:52 smithi175 ceph-mon[89733]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:29:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:53 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:53 smithi179 ceph-mon[67580]: osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992] boot 2024-05-25T06:29:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:53 smithi179 ceph-mon[67580]: osdmap e137: 8 total, 8 up, 8 in 2024-05-25T06:29:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:29:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:53 smithi179 ceph-mon[67580]: osdmap e138: 8 total, 8 up, 8 in 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[84648]: osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992] boot 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[84648]: osdmap e137: 8 total, 8 up, 8 in 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[84648]: osdmap e138: 8 total, 8 up, 8 in 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:29:53] "GET /metrics HTTP/1.1" 200 38272 "" "Prometheus/2.43.0" 2024-05-25T06:29:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:29:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[89733]: osd.5 [v2:172.21.15.179:6808/4226029992,v1:172.21.15.179:6809/4226029992] boot 2024-05-25T06:29:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[89733]: osdmap e137: 8 total, 8 up, 8 in 2024-05-25T06:29:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-25T06:29:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:53.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:53 smithi175 ceph-mon[89733]: osdmap e138: 8 total, 8 up, 8 in 2024-05-25T06:29:54.750 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:54 smithi179 ceph-mon[67580]: pgmap v211: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-25T06:29:54.750 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:54.750 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:54 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:54 smithi175 ceph-mon[84648]: pgmap v211: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-25T06:29:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:54 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:54 smithi175 ceph-mon[89733]: pgmap v211: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-25T06:29:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:54 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:55.825 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:55 smithi179 ceph-mon[67580]: pgmap v213: 161 pgs: 17 active+undersized+degraded, 25 active+undersized, 119 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 60/723 objects degraded (8.299%) 2024-05-25T06:29:55.825 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:55 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 60/723 objects degraded (8.299%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:56.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:55 smithi175 ceph-mon[89733]: pgmap v213: 161 pgs: 17 active+undersized+degraded, 25 active+undersized, 119 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 60/723 objects degraded (8.299%) 2024-05-25T06:29:56.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:55 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 60/723 objects degraded (8.299%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:56.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:55 smithi175 ceph-mon[84648]: pgmap v213: 161 pgs: 17 active+undersized+degraded, 25 active+undersized, 119 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 60/723 objects degraded (8.299%) 2024-05-25T06:29:56.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:55 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 60/723 objects degraded (8.299%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:29:56.118 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:29:56.773 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:29:56.773 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (8m) 47s ago 30m 28.6M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:29:56.773 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (7m) 4s ago 29m 80.0M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 47s ago 29m 62.7M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (9m) 4s ago 34m 453M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (19m) 47s ago 36m 513M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (5m) 47s ago 36m 53.4M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (6m) 4s ago 34m 47.9M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (5m) 47s ago 35m 44.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (8m) 47s ago 30m 16.1M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (8m) 4s ago 30m 15.0M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (3m) 47s ago 34m 53.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (2m) 47s ago 33m 51.4M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (106s) 47s ago 33m 41.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (52s) 47s ago 33m 14.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (30s) 4s ago 32m 43.9M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (9s) 4s ago 32m 12.0M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:29:56.774 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (31m) 4s ago 31m 60.1M 3248M 17.2.0 e1d6a67b021e 2354371ecea2 2024-05-25T06:29:56.775 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (31m) 4s ago 31m 61.8M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:29:56.775 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (8m) 4s ago 30m 67.8M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:29:56.775 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (29m) 47s ago 29m 78.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:29:56.775 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (29m) 4s ago 29m 81.7M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:29:56.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:56 smithi179 ceph-mon[67580]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:56.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:56 smithi179 ceph-mon[67580]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:57.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:56 smithi175 ceph-mon[89733]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:57.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:56 smithi175 ceph-mon[89733]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:57.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:56 smithi175 ceph-mon[84648]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:57.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:56 smithi175 ceph-mon[84648]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:57.168 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 6 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 11 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:29:57.169 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:29:57.526 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "6/8 daemons upgraded", 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading osd daemons", 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:29:57.527 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/841405953' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/841405953' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:57.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:57 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:29:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:29:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/841405953' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:29:58.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:57 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:29:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:58 smithi179 ceph-mon[67580]: from='client.44532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:59.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:58 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/723 objects degraded (8.299%), 17 pgs degraded) 2024-05-25T06:29:59.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:58 smithi175 ceph-mon[89733]: from='client.44532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:59.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:58 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/723 objects degraded (8.299%), 17 pgs degraded) 2024-05-25T06:29:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:58 smithi175 ceph-mon[84648]: from='client.44532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:29:59.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:58 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/723 objects degraded (8.299%), 17 pgs degraded) 2024-05-25T06:29:59.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:29:59.210+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:59.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:59.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:29:59 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:59.847 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:59.847 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:59.848 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:29:59.849 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.850 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:29:59.851 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: Upgrade: osd.6 is safe to restart 2024-05-25T06:29:59.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:29:59 smithi179 ceph-mon[67580]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:30:00.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:00.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:00.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:00.070 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:00.072 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.073 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:00.074 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:00.075 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:00.076 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.077 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:00.078 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: Upgrade: osd.6 is safe to restart 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[84648]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:30:00.079 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: Upgrade: osd.6 is safe to restart 2024-05-25T06:30:00.080 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:29:59 smithi175 ceph-mon[89733]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-25T06:30:00.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:00 smithi179 ceph-mon[67580]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:30:01.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:00 smithi175 ceph-mon[84648]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:30:01.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:00 smithi175 ceph-mon[89733]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:30:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:02 smithi179 ceph-mon[67580]: Upgrade: Updating osd.6 2024-05-25T06:30:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T06:30:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:02 smithi179 ceph-mon[67580]: Deploying daemon osd.6 on smithi179 2024-05-25T06:30:02.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:02 smithi179 ceph-mon[67580]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[84648]: Upgrade: Updating osd.6 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[84648]: Deploying daemon osd.6 on smithi179 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[84648]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[89733]: Upgrade: Updating osd.6 2024-05-25T06:30:02.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:02.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-25T06:30:02.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:02.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[89733]: Deploying daemon osd.6 on smithi179 2024-05-25T06:30:02.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:02 smithi175 ceph-mon[89733]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-25T06:30:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:30:03] "GET /metrics HTTP/1.1" 200 38272 "" "Prometheus/2.43.0" 2024-05-25T06:30:04.525 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:04 smithi179 ceph-mon[67580]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:04.526 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:04 smithi179 systemd[1]: Stopping Ceph osd.6 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:30:04.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:04 smithi175 ceph-mon[84648]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:04.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:04 smithi175 ceph-mon[89733]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:04.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[34784]: 2024-05-25T06:30:04.524+0000 7f9505b62700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:30:04.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[34784]: 2024-05-25T06:30:04.524+0000 7f9505b62700 -1 osd.6 138 *** Got signal Terminated *** 2024-05-25T06:30:04.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[34784]: 2024-05-25T06:30:04.524+0000 7f9505b62700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:30:05.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:05 smithi175 ceph-mon[84648]: osd.6 marked itself down and dead 2024-05-25T06:30:05.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:05 smithi175 ceph-mon[89733]: osd.6 marked itself down and dead 2024-05-25T06:30:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:05 smithi179 ceph-mon[67580]: osd.6 marked itself down and dead 2024-05-25T06:30:06.176 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:05 smithi179 podman[86439]: 2024-05-25 06:30:05.866493222 +0000 UTC m=+1.476502317 container died 2354371ecea2e7ed3c0d64ae1e7349b66cf8101feb59bca83970d0ff1671ae52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, architecture=x86_64, release=754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_CLEAN=True, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-25T06:30:06.526 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:06 smithi179 podman[86439]: 2024-05-25 06:30:06.266628351 +0000 UTC m=+1.876637442 container cleanup 2354371ecea2e7ed3c0d64ae1e7349b66cf8101feb59bca83970d0ff1671ae52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, io.openshift.expose-services=, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, version=8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, name=centos-stream, release=754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-25T06:30:06.526 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:06 smithi179 bash[86439]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6 2024-05-25T06:30:06.527 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:06 smithi179 ceph-mon[67580]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:30:06.527 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:06 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:30:06.527 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:06 smithi179 ceph-mon[67580]: osdmap e139: 8 total, 7 up, 8 in 2024-05-25T06:30:06.527 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:06 smithi179 ceph-mon[67580]: osd.6 marked itself dead as of e139 2024-05-25T06:30:06.527 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:06.527 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:06.527 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:06.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[89733]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[89733]: osdmap e139: 8 total, 7 up, 8 in 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[89733]: osd.6 marked itself dead as of e139 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[84648]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[84648]: osdmap e139: 8 total, 7 up, 8 in 2024-05-25T06:30:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[84648]: osd.6 marked itself dead as of e139 2024-05-25T06:30:06.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:06.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:06.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:06.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:06 smithi179 podman[86457]: 2024-05-25 06:30:06.52676587 +0000 UTC m=+0.653117407 container remove 2354371ecea2e7ed3c0d64ae1e7349b66cf8101feb59bca83970d0ff1671ae52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, release=754, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , name=centos-stream, vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0) 2024-05-25T06:30:07.196 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:06 smithi179 podman[86546]: 2024-05-25 06:30:06.776434965 +0000 UTC m=+0.017788420 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:30:07.196 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:06 smithi179 podman[86546]: 2024-05-25 06:30:06.903128714 +0000 UTC m=+0.144482172 container create d94d3fe759cb2bf4c013db88b5862be6dc5eab91f84906876a16bca22c08f7e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521) 2024-05-25T06:30:07.196 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:07 smithi179 podman[86546]: 2024-05-25 06:30:07.113863302 +0000 UTC m=+0.355216768 container init d94d3fe759cb2bf4c013db88b5862be6dc5eab91f84906876a16bca22c08f7e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521) 2024-05-25T06:30:07.197 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:07 smithi179 podman[86546]: 2024-05-25 06:30:07.117165354 +0000 UTC m=+0.358518816 container start d94d3fe759cb2bf4c013db88b5862be6dc5eab91f84906876a16bca22c08f7e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:30:07.197 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:07 smithi179 podman[86546]: 2024-05-25 06:30:07.197180097 +0000 UTC m=+0.438533554 container attach d94d3fe759cb2bf4c013db88b5862be6dc5eab91f84906876a16bca22c08f7e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux ) 2024-05-25T06:30:07.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:07 smithi179 ceph-mon[67580]: osdmap e140: 8 total, 7 up, 8 in 2024-05-25T06:30:07.535 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:07 smithi179 podman[86546]: 2024-05-25 06:30:07.418974611 +0000 UTC m=+0.660328073 container died d94d3fe759cb2bf4c013db88b5862be6dc5eab91f84906876a16bca22c08f7e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD) 2024-05-25T06:30:07.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:07 smithi175 ceph-mon[84648]: osdmap e140: 8 total, 7 up, 8 in 2024-05-25T06:30:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:07 smithi175 ceph-mon[89733]: osdmap e140: 8 total, 7 up, 8 in 2024-05-25T06:30:08.285 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 podman[86571]: 2024-05-25 06:30:08.017587763 +0000 UTC m=+0.593076028 container remove d94d3fe759cb2bf4c013db88b5862be6dc5eab91f84906876a16bca22c08f7e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-25T06:30:08.285 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.6.service: Deactivated successfully. 2024-05-25T06:30:08.286 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 systemd[1]: Stopped Ceph osd.6 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:30:08.286 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.6.service: Consumed 14.762s CPU time. 2024-05-25T06:30:08.286 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 systemd[1]: Starting Ceph osd.6 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:30:08.590 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 podman[86670]: 2024-05-25 06:30:08.351832935 +0000 UTC m=+0.018326561 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:30:08.590 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 podman[86670]: 2024-05-25 06:30:08.460570772 +0000 UTC m=+0.127064389 container create 69f6c24bdac4ef6355c12962d6f750838623567a036703f68c67199e209dfa5e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2) 2024-05-25T06:30:08.591 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:08 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:08.591 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:08 smithi179 ceph-mon[67580]: pgmap v223: 161 pgs: 8 active+undersized+degraded, 18 peering, 1 stale+active+clean, 12 active+undersized, 122 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 30/723 objects degraded (4.149%) 2024-05-25T06:30:08.591 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:08 smithi179 ceph-mon[67580]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-05-25T06:30:08.591 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:08 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 30/723 objects degraded (4.149%), 8 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[84648]: pgmap v223: 161 pgs: 8 active+undersized+degraded, 18 peering, 1 stale+active+clean, 12 active+undersized, 122 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 30/723 objects degraded (4.149%) 2024-05-25T06:30:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[84648]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-05-25T06:30:08.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 30/723 objects degraded (4.149%), 8 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[89733]: pgmap v223: 161 pgs: 8 active+undersized+degraded, 18 peering, 1 stale+active+clean, 12 active+undersized, 122 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 30/723 objects degraded (4.149%) 2024-05-25T06:30:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[89733]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-05-25T06:30:08.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:08 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 30/723 objects degraded (4.149%), 8 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:09.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 podman[86670]: 2024-05-25 06:30:08.688957615 +0000 UTC m=+0.355451246 container init 69f6c24bdac4ef6355c12962d6f750838623567a036703f68c67199e209dfa5e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-05-25T06:30:09.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 podman[86670]: 2024-05-25 06:30:08.692576794 +0000 UTC m=+0.359070410 container start 69f6c24bdac4ef6355c12962d6f750838623567a036703f68c67199e209dfa5e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-25T06:30:09.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:08 smithi179 podman[86670]: 2024-05-25 06:30:08.771813814 +0000 UTC m=+0.438307446 container attach 69f6c24bdac4ef6355c12962d6f750838623567a036703f68c67199e209dfa5e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, GIT_CLEAN=True) 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate[86686]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate[86686]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 bash[86670]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 bash[86670]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 bash[86670]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 bash[86670]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 bash[86670]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 bash[86670]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 bash[86670]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate[86686]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-25T06:30:09.476 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate[86686]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-25T06:30:09.477 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate[86686]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-25T06:30:09.477 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate[86686]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-25T06:30:09.477 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate[86686]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-25T06:30:09.477 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 podman[86670]: 2024-05-25 06:30:09.161455881 +0000 UTC m=+0.827949505 container died 69f6c24bdac4ef6355c12962d6f750838623567a036703f68c67199e209dfa5e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:30:10.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 podman[86801]: 2024-05-25 06:30:09.742523372 +0000 UTC m=+0.572417970 container remove 69f6c24bdac4ef6355c12962d6f750838623567a036703f68c67199e209dfa5e (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:30:10.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 podman[86860]: 2024-05-25 06:30:09.8573197 +0000 UTC m=+0.017946216 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:30:10.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:09 smithi179 podman[86860]: 2024-05-25 06:30:09.96037856 +0000 UTC m=+0.121005053 container create 8f5beea10bda4d378c3e93778ffa2f47125a501dfc5045e51e75e1a0d32e22d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-05-25T06:30:10.333 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:10 smithi179 podman[86860]: 2024-05-25 06:30:10.179366207 +0000 UTC m=+0.339992707 container init 8f5beea10bda4d378c3e93778ffa2f47125a501dfc5045e51e75e1a0d32e22d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-05-25T06:30:10.333 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:10 smithi179 podman[86860]: 2024-05-25 06:30:10.182635014 +0000 UTC m=+0.343261514 container start 8f5beea10bda4d378c3e93778ffa2f47125a501dfc5045e51e75e1a0d32e22d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521) 2024-05-25T06:30:10.333 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:10 smithi179 ceph-osd[86878]: -- 172.21.15.179:0/2521537951 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563224cd7680 con 0x563224ca1400 2024-05-25T06:30:10.333 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:10 smithi179 bash[86860]: 8f5beea10bda4d378c3e93778ffa2f47125a501dfc5045e51e75e1a0d32e22d4 2024-05-25T06:30:10.333 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:10 smithi179 systemd[1]: Started Ceph osd.6 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:30:10.733 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:10 smithi179 ceph-mon[67580]: pgmap v224: 161 pgs: 12 active+undersized+degraded, 18 peering, 23 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 54/723 objects degraded (7.469%) 2024-05-25T06:30:10.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:10 smithi175 ceph-mon[84648]: pgmap v224: 161 pgs: 12 active+undersized+degraded, 18 peering, 23 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 54/723 objects degraded (7.469%) 2024-05-25T06:30:10.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:10 smithi175 ceph-mon[89733]: pgmap v224: 161 pgs: 12 active+undersized+degraded, 18 peering, 23 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 54/723 objects degraded (7.469%) 2024-05-25T06:30:11.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:10 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:30:10.734+0000 7ff67597f7c0 -1 Falling back to public interface 2024-05-25T06:30:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:11.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:11 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:11.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:11.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:11.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:11 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:11.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:11 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:12.704 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:12 smithi179 ceph-mon[67580]: pgmap v225: 161 pgs: 16 active+undersized+degraded, 3 peering, 34 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-05-25T06:30:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:12 smithi175 ceph-mon[84648]: pgmap v225: 161 pgs: 16 active+undersized+degraded, 3 peering, 34 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-05-25T06:30:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:12 smithi175 ceph-mon[89733]: pgmap v225: 161 pgs: 16 active+undersized+degraded, 3 peering, 34 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-05-25T06:30:13.727 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:13 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 83/723 objects degraded (11.480%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:13.727 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:13 smithi179 ceph-mon[67580]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-25T06:30:13.727 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:30:13.438+0000 7ff67597f7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-05-25T06:30:13.727 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:30:13.465+0000 7ff67597f7c0 -1 osd.6 139 log_to_monitors true 2024-05-25T06:30:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:13 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 83/723 objects degraded (11.480%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:13.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:13 smithi175 ceph-mon[84648]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-25T06:30:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:13 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 83/723 objects degraded (11.480%), 17 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:13.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:13 smithi175 ceph-mon[89733]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-25T06:30:13.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:30:13] "GET /metrics HTTP/1.1" 200 38276 "" "Prometheus/2.43.0" 2024-05-25T06:30:14.535 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:30:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:30:14.429+0000 7ff666dd6700 -1 osd.6 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:30:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:14 smithi179 ceph-mon[67580]: pgmap v226: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-05-25T06:30:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:14 smithi179 ceph-mon[67580]: from='osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T06:30:14.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:14 smithi179 ceph-mon[67580]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T06:30:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:14 smithi175 ceph-mon[84648]: pgmap v226: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-05-25T06:30:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:14 smithi175 ceph-mon[84648]: from='osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T06:30:14.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:14 smithi175 ceph-mon[84648]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T06:30:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:14 smithi175 ceph-mon[89733]: pgmap v226: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-05-25T06:30:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:14 smithi175 ceph-mon[89733]: from='osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T06:30:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:14 smithi175 ceph-mon[89733]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-25T06:30:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:15 smithi179 ceph-mon[67580]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-25T06:30:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:15 smithi179 ceph-mon[67580]: from='osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:15 smithi179 ceph-mon[67580]: osdmap e141: 8 total, 7 up, 8 in 2024-05-25T06:30:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:15 smithi179 ceph-mon[67580]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:15.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:15 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[84648]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[84648]: from='osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[84648]: osdmap e141: 8 total, 7 up, 8 in 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[84648]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[89733]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[89733]: from='osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[89733]: osdmap e141: 8 total, 7 up, 8 in 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[89733]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:15 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:16.418 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: pgmap v228: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817] boot 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: osdmap e142: 8 total, 8 up, 8 in 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: pgmap v228: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817] boot 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: osdmap e142: 8 total, 8 up, 8 in 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:30:16.419 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:16.420 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:16.420 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[84648]: osdmap e143: 8 total, 8 up, 8 in 2024-05-25T06:30:16.716 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:16.717 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:16.717 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:16 smithi175 ceph-mon[89733]: osdmap e143: 8 total, 8 up, 8 in 2024-05-25T06:30:16.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: pgmap v228: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-05-25T06:30:16.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:30:16.729 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: osd.6 [v2:172.21.15.179:6816/1557935817,v1:172.21.15.179:6817/1557935817] boot 2024-05-25T06:30:16.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: osdmap e142: 8 total, 8 up, 8 in 2024-05-25T06:30:16.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-25T06:30:16.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:16.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:16.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:16 smithi179 ceph-mon[67580]: osdmap e143: 8 total, 8 up, 8 in 2024-05-25T06:30:18.753 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:18 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:18.753 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:18 smithi179 ceph-mon[67580]: pgmap v231: 161 pgs: 1 peering, 11 active+undersized+degraded, 25 active+undersized, 124 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 55/723 objects degraded (7.607%) 2024-05-25T06:30:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:18 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:18.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:18 smithi175 ceph-mon[89733]: pgmap v231: 161 pgs: 1 peering, 11 active+undersized+degraded, 25 active+undersized, 124 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 55/723 objects degraded (7.607%) 2024-05-25T06:30:18.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:18 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:18.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:18 smithi175 ceph-mon[84648]: pgmap v231: 161 pgs: 1 peering, 11 active+undersized+degraded, 25 active+undersized, 124 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 55/723 objects degraded (7.607%) 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: pgmap v232: 161 pgs: 1 peering, 7 active+undersized+degraded, 15 active+undersized, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 31/723 objects degraded (4.288%) 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: pgmap v233: 161 pgs: 1 peering, 7 active+undersized+degraded, 15 active+undersized, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 31/723 objects degraded (4.288%) 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:20.244 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: pgmap v232: 161 pgs: 1 peering, 7 active+undersized+degraded, 15 active+undersized, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 31/723 objects degraded (4.288%) 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: pgmap v233: 161 pgs: 1 peering, 7 active+undersized+degraded, 15 active+undersized, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 31/723 objects degraded (4.288%) 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:20.245 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:20 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:30:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:20.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: pgmap v232: 161 pgs: 1 peering, 7 active+undersized+degraded, 15 active+undersized, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 31/723 objects degraded (4.288%) 2024-05-25T06:30:20.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: pgmap v233: 161 pgs: 1 peering, 7 active+undersized+degraded, 15 active+undersized, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 31/723 objects degraded (4.288%) 2024-05-25T06:30:20.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:20.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:30:20.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:20.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:20.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:20 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 31/723 objects degraded (4.288%), 7 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 31/723 objects degraded (4.288%), 7 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.321 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:30:21.120+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:21.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:21.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:21.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:30:21.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:21.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:21.327 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:21 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:30:21.132+0000 7f7edf5ab700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-25T06:30:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 31/723 objects degraded (4.288%), 7 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:21.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:22.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:22 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:22.535 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:22.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:22.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:22.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:22.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:22.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:22.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:22.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.540 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.540 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:22 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:22.540 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:22 smithi179 ceph-mon[67580]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-25T06:30:22.540 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:22 smithi179 ceph-mon[67580]: pgmap v234: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:22.572 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:22.573 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.574 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.575 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:22.576 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:22.577 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[84648]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[84648]: pgmap v234: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-05-25T06:30:22.578 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:22.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:22.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:22.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:22.579 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:22.579 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:22.579 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[89733]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-25T06:30:22.579 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:22 smithi175 ceph-mon[89733]: pgmap v234: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-05-25T06:30:23.491 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:23 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/723 objects degraded (4.288%), 7 pgs degraded) 2024-05-25T06:30:23.492 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:23 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/723 objects degraded (4.288%), 7 pgs degraded) 2024-05-25T06:30:23.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:23 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/723 objects degraded (4.288%), 7 pgs degraded) 2024-05-25T06:30:23.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:30:23] "GET /metrics HTTP/1.1" 200 38288 "" "Prometheus/2.43.0" 2024-05-25T06:30:24.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:24 smithi179 ceph-mon[67580]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 0 op/s 2024-05-25T06:30:24.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:24 smithi175 ceph-mon[84648]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 0 op/s 2024-05-25T06:30:24.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:24 smithi175 ceph-mon[89733]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 0 op/s 2024-05-25T06:30:26.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:26 smithi179 ceph-mon[67580]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:30:26.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:26 smithi175 ceph-mon[84648]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:30:26.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:26 smithi175 ceph-mon[89733]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:30:27.886 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:30:28.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:28 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:28.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:28 smithi179 ceph-mon[67580]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-05-25T06:30:28.540 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:28 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:28.540 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:28 smithi175 ceph-mon[84648]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-05-25T06:30:28.540 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:28 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:28.540 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:28 smithi175 ceph-mon[89733]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-05-25T06:30:28.540 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:30:28.540 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (8m) 79s ago 30m 28.6M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:30:28.540 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (8m) 13s ago 30m 81.9M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:30:28.540 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 79s ago 29m 62.7M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (9m) 13s ago 35m 454M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (19m) 79s ago 36m 513M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (6m) 79s ago 36m 53.4M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (6m) 13s ago 35m 49.0M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (5m) 79s ago 35m 44.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (9m) 79s ago 31m 16.1M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (9m) 13s ago 31m 14.7M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (3m) 79s ago 34m 53.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (3m) 79s ago 34m 51.4M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (2m) 79s ago 34m 41.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (84s) 79s ago 33m 14.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (62s) 13s ago 33m 44.4M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (40s) 13s ago 32m 43.1M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:30:28.541 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (18s) 13s ago 32m 11.9M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 8f5beea10bda 2024-05-25T06:30:28.542 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (32m) 13s ago 32m 63.2M 3248M 17.2.0 e1d6a67b021e 84f861f0cfe2 2024-05-25T06:30:28.542 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (8m) 13s ago 31m 67.8M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:30:28.542 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (29m) 79s ago 29m 78.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:30:28.542 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (29m) 13s ago 29m 81.7M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:30:28.937 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:30:28.937 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 7 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:30:28.938 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:30:28.939 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-25T06:30:28.939 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 12 2024-05-25T06:30:28.939 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:30:28.939 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:30:29.275 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:30:29.275 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:30:29.275 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:30:29.275 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-25T06:30:29.275 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:30:29.275 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "7/8 daemons upgraded", 2024-05-25T06:30:29.276 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Currently upgrading osd daemons", 2024-05-25T06:30:29.276 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:30:29.276 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:30:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:29 smithi179 ceph-mon[67580]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:29 smithi179 ceph-mon[67580]: from='client.54487 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:29 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1590979705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:29 smithi175 ceph-mon[89733]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:29 smithi175 ceph-mon[89733]: from='client.54487 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:29 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1590979705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:29 smithi175 ceph-mon[84648]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:29 smithi175 ceph-mon[84648]: from='client.54487 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:29 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1590979705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:30 smithi179 ceph-mon[67580]: from='client.44556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:30 smithi179 ceph-mon[67580]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:30:30.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:30 smithi179 ceph-mon[67580]: from='client.54505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:30.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:30 smithi175 ceph-mon[89733]: from='client.44556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:30 smithi175 ceph-mon[89733]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:30:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:30 smithi175 ceph-mon[89733]: from='client.54505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:30 smithi175 ceph-mon[84648]: from='client.44556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:30 smithi175 ceph-mon[84648]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:30:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:30 smithi175 ceph-mon[84648]: from='client.54505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:30:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:31 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:32.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:31 smithi179 ceph-mon[67580]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:31 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:32.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:31 smithi175 ceph-mon[84648]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:31 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:32.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:31 smithi175 ceph-mon[89733]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:30:33] "GET /metrics HTTP/1.1" 200 38288 "" "Prometheus/2.43.0" 2024-05-25T06:30:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:34 smithi179 ceph-mon[67580]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:34 smithi175 ceph-mon[84648]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:34.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:34 smithi175 ceph-mon[89733]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:30:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:36 smithi179 ceph-mon[67580]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:30:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:36.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:36.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[84648]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:30:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:36.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[89733]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:30:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:36.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:37.720 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:37 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:37.721 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:37 smithi179 ceph-mon[67580]: Upgrade: osd.7 is safe to restart 2024-05-25T06:30:37.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:37 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:37 smithi175 ceph-mon[84648]: Upgrade: osd.7 is safe to restart 2024-05-25T06:30:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:37 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-25T06:30:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:37 smithi175 ceph-mon[89733]: Upgrade: osd.7 is safe to restart 2024-05-25T06:30:38.587 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:38 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:38.587 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:38 smithi179 ceph-mon[67580]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 33 op/s 2024-05-25T06:30:38.587 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:38 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:38.587 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:38 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T06:30:38.587 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:38 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[84648]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 33 op/s 2024-05-25T06:30:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T06:30:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[89733]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 33 op/s 2024-05-25T06:30:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-25T06:30:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:38 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:39.628 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:39 smithi179 ceph-mon[67580]: Upgrade: Updating osd.7 2024-05-25T06:30:39.628 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:39 smithi179 ceph-mon[67580]: Deploying daemon osd.7 on smithi179 2024-05-25T06:30:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:39 smithi175 ceph-mon[84648]: Upgrade: Updating osd.7 2024-05-25T06:30:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:39 smithi175 ceph-mon[84648]: Deploying daemon osd.7 on smithi179 2024-05-25T06:30:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:39 smithi175 ceph-mon[89733]: Upgrade: Updating osd.7 2024-05-25T06:30:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:39 smithi175 ceph-mon[89733]: Deploying daemon osd.7 on smithi179 2024-05-25T06:30:40.762 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:40 smithi179 ceph-mon[67580]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 51 op/s 2024-05-25T06:30:40.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:40 smithi175 ceph-mon[84648]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 51 op/s 2024-05-25T06:30:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:40 smithi175 ceph-mon[89733]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 51 op/s 2024-05-25T06:30:41.518 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:41 smithi179 systemd[1]: Stopping Ceph osd.7 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:30:41.518 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[37685]: 2024-05-25T06:30:41.349+0000 7f04bbda1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:30:41.518 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[37685]: 2024-05-25T06:30:41.349+0000 7f04bbda1700 -1 osd.7 143 *** Got signal Terminated *** 2024-05-25T06:30:41.518 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:41 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[37685]: 2024-05-25T06:30:41.349+0000 7f04bbda1700 -1 osd.7 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:30:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:41 smithi179 ceph-mon[67580]: osd.7 marked itself down and dead 2024-05-25T06:30:41.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:41 smithi175 ceph-mon[84648]: osd.7 marked itself down and dead 2024-05-25T06:30:41.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:41 smithi175 ceph-mon[89733]: osd.7 marked itself down and dead 2024-05-25T06:30:42.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:42 smithi179 podman[90922]: 2024-05-25 06:30:42.034708063 +0000 UTC m=+0.779598746 container died 84f861f0cfe29d2ee7bfaa74fea6f9dbc359146a0a736ae25370762ffa99f736 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, GIT_BRANCH=HEAD, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-25T06:30:42.676 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:42 smithi179 ceph-mon[67580]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:30:42.676 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:42 smithi179 ceph-mon[67580]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:30:42.676 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:42 smithi179 ceph-mon[67580]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-25T06:30:42.676 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:42 smithi179 ceph-mon[67580]: osdmap e144: 8 total, 7 up, 8 in 2024-05-25T06:30:42.676 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:42 smithi179 podman[90922]: 2024-05-25 06:30:42.449547889 +0000 UTC m=+1.194438570 container cleanup 84f861f0cfe29d2ee7bfaa74fea6f9dbc359146a0a736ae25370762ffa99f736 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.buildah.version=1.19.8) 2024-05-25T06:30:42.677 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:42 smithi179 bash[90922]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[89733]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[89733]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[89733]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[89733]: osdmap e144: 8 total, 7 up, 8 in 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[84648]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[84648]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[84648]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-25T06:30:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:42 smithi175 ceph-mon[84648]: osdmap e144: 8 total, 7 up, 8 in 2024-05-25T06:30:43.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:42 smithi179 podman[90938]: 2024-05-25 06:30:42.676407719 +0000 UTC m=+0.645148907 container remove 84f861f0cfe29d2ee7bfaa74fea6f9dbc359146a0a736ae25370762ffa99f736 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-05-25T06:30:43.366 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:43 smithi179 podman[91029]: 2024-05-25 06:30:43.016664765 +0000 UTC m=+0.017400386 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:30:43.366 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:43 smithi179 podman[91029]: 2024-05-25 06:30:43.135929081 +0000 UTC m=+0.136664705 container create ae47e777daad5a5062267d104c427ce55b7b0606a46f6bfe170c272fd585433d (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-25T06:30:43.366 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:43 smithi179 podman[91029]: 2024-05-25 06:30:43.363100403 +0000 UTC m=+0.363836005 container init ae47e777daad5a5062267d104c427ce55b7b0606a46f6bfe170c272fd585433d (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-25T06:30:43.366 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:43 smithi179 podman[91029]: 2024-05-25 06:30:43.366618007 +0000 UTC m=+0.367353613 container start ae47e777daad5a5062267d104c427ce55b7b0606a46f6bfe170c272fd585433d (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, GIT_BRANCH=HEAD) 2024-05-25T06:30:43.659 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:43 smithi179 ceph-mon[67580]: osdmap e145: 8 total, 7 up, 8 in 2024-05-25T06:30:43.659 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:43 smithi179 podman[91029]: 2024-05-25 06:30:43.471691638 +0000 UTC m=+0.472427256 container attach ae47e777daad5a5062267d104c427ce55b7b0606a46f6bfe170c272fd585433d (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-ea0684c) 2024-05-25T06:30:43.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:43 smithi175 ceph-mon[89733]: osdmap e145: 8 total, 7 up, 8 in 2024-05-25T06:30:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:43 smithi175 ceph-mon[84648]: osdmap e145: 8 total, 7 up, 8 in 2024-05-25T06:30:43.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:30:43] "GET /metrics HTTP/1.1" 200 38288 "" "Prometheus/2.43.0" 2024-05-25T06:30:44.008 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:43 smithi179 podman[91029]: 2024-05-25 06:30:43.65993925 +0000 UTC m=+0.660674862 container died ae47e777daad5a5062267d104c427ce55b7b0606a46f6bfe170c272fd585433d (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:30:44.762 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:44 smithi179 ceph-mon[67580]: pgmap v247: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 0 B/s wr, 95 op/s 2024-05-25T06:30:44.763 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:44 smithi179 podman[91054]: 2024-05-25 06:30:44.32582864 +0000 UTC m=+0.659714558 container remove ae47e777daad5a5062267d104c427ce55b7b0606a46f6bfe170c272fd585433d (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-05-25T06:30:44.763 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:44 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.7.service: Deactivated successfully. 2024-05-25T06:30:44.763 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:44 smithi179 systemd[1]: Stopped Ceph osd.7 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:30:44.763 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:44 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.7.service: Consumed 17.216s CPU time. 2024-05-25T06:30:44.763 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:44 smithi179 systemd[1]: Starting Ceph osd.7 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:30:44.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:44 smithi175 ceph-mon[89733]: pgmap v247: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 0 B/s wr, 95 op/s 2024-05-25T06:30:44.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:44 smithi175 ceph-mon[84648]: pgmap v247: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 0 B/s wr, 95 op/s 2024-05-25T06:30:45.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:44 smithi179 podman[91154]: 2024-05-25 06:30:44.665477383 +0000 UTC m=+0.017471962 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:30:45.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:44 smithi179 podman[91154]: 2024-05-25 06:30:44.868696635 +0000 UTC m=+0.220691204 container create 41925872c7ec7def9ffce4e4d2503a4f568d650d6de173211dbbee7b89414fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521) 2024-05-25T06:30:45.530 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 podman[91154]: 2024-05-25 06:30:45.079617531 +0000 UTC m=+0.431612085 container init 41925872c7ec7def9ffce4e4d2503a4f568d650d6de173211dbbee7b89414fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:30:45.530 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 podman[91154]: 2024-05-25 06:30:45.08365872 +0000 UTC m=+0.435653297 container start 41925872c7ec7def9ffce4e4d2503a4f568d650d6de173211dbbee7b89414fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c) 2024-05-25T06:30:45.530 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 podman[91154]: 2024-05-25 06:30:45.154424861 +0000 UTC m=+0.506419427 container attach 41925872c7ec7def9ffce4e4d2503a4f568d650d6de173211dbbee7b89414fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, io.buildah.version=1.33.5) 2024-05-25T06:30:45.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:45 smithi179 ceph-mon[67580]: pgmap v248: 161 pgs: 9 active+undersized+degraded, 13 stale+active+clean, 27 active+undersized, 112 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 82 op/s; 27/723 objects degraded (3.734%) 2024-05-25T06:30:45.785 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate[91169]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-25T06:30:45.785 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate[91169]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-25T06:30:45.785 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 bash[91154]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 bash[91154]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 bash[91154]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 bash[91154]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 bash[91154]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 bash[91154]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 bash[91154]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate[91169]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate[91169]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate[91169]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate[91169]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate[91169]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-25T06:30:45.786 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:45 smithi179 podman[91154]: 2024-05-25 06:30:45.557254361 +0000 UTC m=+0.909248933 container died 41925872c7ec7def9ffce4e4d2503a4f568d650d6de173211dbbee7b89414fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240521, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-25T06:30:45.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:45 smithi175 ceph-mon[89733]: pgmap v248: 161 pgs: 9 active+undersized+degraded, 13 stale+active+clean, 27 active+undersized, 112 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 82 op/s; 27/723 objects degraded (3.734%) 2024-05-25T06:30:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:45 smithi175 ceph-mon[84648]: pgmap v248: 161 pgs: 9 active+undersized+degraded, 13 stale+active+clean, 27 active+undersized, 112 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 82 op/s; 27/723 objects degraded (3.734%) 2024-05-25T06:30:46.512 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 podman[91284]: 2024-05-25 06:30:46.158351231 +0000 UTC m=+0.590324744 container remove 41925872c7ec7def9ffce4e4d2503a4f568d650d6de173211dbbee7b89414fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-25T06:30:46.512 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 podman[91344]: 2024-05-25 06:30:46.282479891 +0000 UTC m=+0.017497182 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:30:46.512 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 podman[91344]: 2024-05-25 06:30:46.392985031 +0000 UTC m=+0.128002363 container create 771fbf01687f07bc186852a6fb374f800341e4b6333a29b25b16e47de74b33a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-05-25T06:30:46.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:46 smithi175 ceph-mon[89733]: Health check failed: Degraded data redundancy: 27/723 objects degraded (3.734%), 9 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:46 smithi175 ceph-mon[84648]: Health check failed: Degraded data redundancy: 27/723 objects degraded (3.734%), 9 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:46 smithi179 ceph-mon[67580]: Health check failed: Degraded data redundancy: 27/723 objects degraded (3.734%), 9 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:47.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 podman[91344]: 2024-05-25 06:30:46.637741052 +0000 UTC m=+0.372758329 container init 771fbf01687f07bc186852a6fb374f800341e4b6333a29b25b16e47de74b33a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-05-25T06:30:47.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 podman[91344]: 2024-05-25 06:30:46.641736285 +0000 UTC m=+0.376753573 container start 771fbf01687f07bc186852a6fb374f800341e4b6333a29b25b16e47de74b33a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=) 2024-05-25T06:30:47.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 ceph-osd[91362]: -- 172.21.15.179:0/402598692 <== mon.2 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562d17951680 con 0x562d1791b000 2024-05-25T06:30:47.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 bash[91344]: 771fbf01687f07bc186852a6fb374f800341e4b6333a29b25b16e47de74b33a9 2024-05-25T06:30:47.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:46 smithi179 systemd[1]: Started Ceph osd.7 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:30:47.535 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:47 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:30:47.180+0000 7f4885b187c0 -1 Falling back to public interface 2024-05-25T06:30:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:47 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:48.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:47 smithi179 ceph-mon[67580]: pgmap v249: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 101 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[84648]: pgmap v249: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 101 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:47 smithi175 ceph-mon[89733]: pgmap v249: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 101 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:50.159 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:30:49.894+0000 7f4885b187c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-05-25T06:30:50.159 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:49 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:30:49.931+0000 7f4885b187c0 -1 osd.7 143 log_to_monitors true 2024-05-25T06:30:50.535 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:30:50 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:30:50.368+0000 7f4876f6f700 -1 osd.7 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-25T06:30:50.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:50 smithi179 ceph-mon[67580]: pgmap v250: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 0 B/s wr, 87 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:50.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:50 smithi179 ceph-mon[67580]: from='osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T06:30:50.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:50 smithi179 ceph-mon[67580]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T06:30:50.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:50 smithi175 ceph-mon[84648]: pgmap v250: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 0 B/s wr, 87 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:50 smithi175 ceph-mon[84648]: from='osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T06:30:50.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:50 smithi175 ceph-mon[84648]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T06:30:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:50 smithi175 ceph-mon[89733]: pgmap v250: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 0 B/s wr, 87 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:50 smithi175 ceph-mon[89733]: from='osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T06:30:50.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:50 smithi175 ceph-mon[89733]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-25T06:30:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:51 smithi179 ceph-mon[67580]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-25T06:30:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:51 smithi179 ceph-mon[67580]: from='osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:51 smithi179 ceph-mon[67580]: osdmap e146: 8 total, 7 up, 8 in 2024-05-25T06:30:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:51 smithi179 ceph-mon[67580]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:51.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:51.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[84648]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[84648]: from='osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[84648]: osdmap e146: 8 total, 7 up, 8 in 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[84648]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[89733]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[89733]: from='osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[89733]: osdmap e146: 8 total, 7 up, 8 in 2024-05-25T06:30:51.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[89733]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-05-25T06:30:51.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:51.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: pgmap v252: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 0 B/s wr, 75 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093] boot 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: osdmap e147: 8 total, 8 up, 8 in 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 26 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:52.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[84648]: osdmap e148: 8 total, 8 up, 8 in 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: pgmap v252: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 0 B/s wr, 75 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093] boot 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: osdmap e147: 8 total, 8 up, 8 in 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 26 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:52.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:52 smithi175 ceph-mon[89733]: osdmap e148: 8 total, 8 up, 8 in 2024-05-25T06:30:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: pgmap v252: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 0 B/s wr, 75 op/s; 95/723 objects degraded (13.140%) 2024-05-25T06:30:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-25T06:30:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: osd.7 [v2:172.21.15.179:6824/3400564093,v1:172.21.15.179:6825/3400564093] boot 2024-05-25T06:30:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: osdmap e147: 8 total, 8 up, 8 in 2024-05-25T06:30:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-25T06:30:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:52.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 26 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:52.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:52 smithi179 ceph-mon[67580]: osdmap e148: 8 total, 8 up, 8 in 2024-05-25T06:30:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:53 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:53.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:53 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:53.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:30:53] "GET /metrics HTTP/1.1" 200 38219 "" "Prometheus/2.43.0" 2024-05-25T06:30:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:53.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:53 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:54.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:54 smithi179 ceph-mon[67580]: pgmap v255: 161 pgs: 16 peering, 19 active+undersized+degraded, 34 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-05-25T06:30:54.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:54 smithi175 ceph-mon[84648]: pgmap v255: 161 pgs: 16 peering, 19 active+undersized+degraded, 34 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-05-25T06:30:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:54 smithi175 ceph-mon[89733]: pgmap v255: 161 pgs: 16 peering, 19 active+undersized+degraded, 34 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-05-25T06:30:56.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: pgmap v256: 161 pgs: 16 peering, 12 active+undersized+degraded, 15 active+undersized, 118 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%); 0 B/s, 0 objects/s recovering 2024-05-25T06:30:56.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:56.756 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: pgmap v256: 161 pgs: 16 peering, 12 active+undersized+degraded, 15 active+undersized, 118 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%); 0 B/s, 0 objects/s recovering 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.757 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:56.758 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:56 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: pgmap v256: 161 pgs: 16 peering, 12 active+undersized+degraded, 15 active+undersized, 118 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%); 0 B/s, 0 objects/s recovering 2024-05-25T06:30:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:56.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:56.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:56 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:57 smithi179 ceph-mon[67580]: pgmap v257: 161 pgs: 16 peering, 145 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 344 B/s rd, 0 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:30:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:57 smithi179 ceph-mon[67580]: pgmap v258: 161 pgs: 16 peering, 145 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 0 B/s, 7 objects/s recovering 2024-05-25T06:30:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:57 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:30:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:57 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:30:57.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:57 smithi179 ceph-mon[67580]: Health check update: Degraded data redundancy: 56/723 objects degraded (7.746%), 12 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:57.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:57 smithi179 ceph-mon[67580]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 12 pgs degraded) 2024-05-25T06:30:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[89733]: pgmap v257: 161 pgs: 16 peering, 145 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 344 B/s rd, 0 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:30:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[89733]: pgmap v258: 161 pgs: 16 peering, 145 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 0 B/s, 7 objects/s recovering 2024-05-25T06:30:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:30:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:30:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[89733]: Health check update: Degraded data redundancy: 56/723 objects degraded (7.746%), 12 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:57.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[89733]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 12 pgs degraded) 2024-05-25T06:30:57.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[84648]: pgmap v257: 161 pgs: 16 peering, 145 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 344 B/s rd, 0 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:30:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[84648]: pgmap v258: 161 pgs: 16 peering, 145 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 0 B/s, 7 objects/s recovering 2024-05-25T06:30:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:30:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:30:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[84648]: Health check update: Degraded data redundancy: 56/723 objects degraded (7.746%), 12 pgs degraded (PG_DEGRADED) 2024-05-25T06:30:57.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:57 smithi175 ceph-mon[84648]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 12 pgs degraded) 2024-05-25T06:30:58.320 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:30:58.128+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:58.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:30:58.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:58.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:58.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:58.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:58.321 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:58.322 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:58.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:58.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:58.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:58.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:58.323 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:58.324 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:58.325 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:58.326 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:58.327 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:58.327 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:58.327 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:30:58 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-25T06:30:58.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-25T06:30:58.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:58 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-25T06:30:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-25T06:30:58.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.822 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-25T06:30:58.823 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-25T06:30:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-25T06:30:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-25T06:30:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-25T06:30:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-25T06:30:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-25T06:30:58.824 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:58 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-25T06:30:59.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:30:59.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:59.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:59.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:59.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all osd 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: Upgrade: Setting require_osd_release to 19 squid 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 295 B/s rd, 0 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: osdmap e149: 8 total, 8 up, 8 in 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:30:59.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:59.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:30:59 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:59.820 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:59.821 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:59.822 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:59.823 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.824 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.825 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:30:59.826 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:30:59.827 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:30:59.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:30:59.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:30:59.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:30:59.828 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:30:59.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:59.828 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all osd 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: Upgrade: Setting require_osd_release to 19 squid 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 295 B/s rd, 0 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: osdmap e149: 8 total, 8 up, 8 in 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.829 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:30:59.830 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:30:59.831 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:30:59.832 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all osd 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: Upgrade: Setting require_osd_release to 19 squid 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 295 B/s rd, 0 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: osdmap e149: 8 total, 8 up, 8 in 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.834 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:59.835 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:30:59.836 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.837 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:30:59.838 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:30:59 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:00.334 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:31:00.697 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mds 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nfs 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Upgrade: Finalizing container_image settings 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Upgrade: Complete! 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 500 B/s rd, 0 op/s; 0 B/s, 8 objects/s recovering 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:00.698 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mds 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nfs 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Upgrade: Finalizing container_image settings 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Upgrade: Complete! 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 500 B/s rd, 0 op/s; 0 B/s, 8 objects/s recovering 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:00.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:00 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mds 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nfs 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Upgrade: Finalizing container_image settings 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Upgrade: Complete! 2024-05-25T06:31:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 0 B/s, 6 objects/s recovering 2024-05-25T06:31:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 500 B/s rd, 0 op/s; 0 B/s, 8 objects/s recovering 2024-05-25T06:31:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:00.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:00 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:01.750 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:01 smithi175 ceph-mon[89733]: from='client.54517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:01.751 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:01 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:01.751 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:01 smithi175 ceph-mon[84648]: from='client.54517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:01.751 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:01 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:01.751 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:31:01.573+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:01.751 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:01.752 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:31:01.753 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:01.754 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:01.755 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:01.756 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:01.756 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:01.756 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:01.756 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:01 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:01 smithi179 ceph-mon[67580]: from='client.54517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:01.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:01 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:02 smithi179 ceph-mon[67580]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-05-25T06:31:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:02 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:02.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[84648]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[89733]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:02.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:02.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:02.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:02.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:02.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:02 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:03.402 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:31:03.402 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (9m) 114s ago 31m 28.6M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (8m) 12s ago 30m 82.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 114s ago 30m 62.7M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (10m) 12s ago 35m 454M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (20m) 114s ago 37m 513M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (6m) 114s ago 37m 53.4M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (7m) 12s ago 35m 48.0M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (6m) 114s ago 36m 44.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (10m) 114s ago 31m 16.1M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (9m) 12s ago 31m 14.8M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (4m) 114s ago 35m 53.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (3m) 114s ago 35m 51.4M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (2m) 114s ago 34m 41.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (119s) 114s ago 34m 14.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:31:03.403 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (97s) 12s ago 33m 47.5M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:31:03.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (75s) 12s ago 33m 45.0M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:31:03.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (53s) 12s ago 33m 43.6M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 8f5beea10bda 2024-05-25T06:31:03.404 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (17s) 12s ago 32m 12.6M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 771fbf01687f 2024-05-25T06:31:03.404 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (9m) 12s ago 32m 67.1M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:31:03.404 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (30m) 114s ago 30m 78.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:31:03.404 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (30m) 12s ago 30m 82.1M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:31:03.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:03.775 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:03.775 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:03.775 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:03.776 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:03.777 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.778 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[84648]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[84648]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:31:03.779 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:31:03] "GET /metrics HTTP/1.1" 200 38219 "" "Prometheus/2.43.0" 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:03.780 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.781 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:31:03.782 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:03.783 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[89733]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[89733]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:03.784 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:03 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:03 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:03.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:03.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:03 smithi179 ceph-mon[67580]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:03 smithi179 ceph-mon[67580]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:03 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:03.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:03 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:04.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-05-25T06:31:04.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:04 smithi179 ceph-mon[67580]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:04.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:04 smithi175 ceph-mon[84648]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:04.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:04 smithi175 ceph-mon[89733]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:05.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:31:05.290+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:05.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:05.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:05.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:05.516 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:05.517 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:05.518 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:05.518 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:05.518 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:05.518 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:05.518 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:05.518 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:05.518 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:05.519 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:05.520 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:05.521 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:05.521 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:05.521 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:05.521 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:05.521 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:05.521 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:05.521 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:05 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:05 smithi179 ceph-mon[67580]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:31:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:05 smithi175 ceph-mon[84648]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:31:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:05.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:05 smithi175 ceph-mon[89733]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:31:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:06.766 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:06 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:06.766 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:06.766 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:06.766 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:06.767 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:06.768 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.769 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:06 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:06.770 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:06.771 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:06.772 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:06.773 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:06.774 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:06.775 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.776 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.776 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:06 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:06.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:06.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:06.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:06.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:06.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:06.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:06.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:06.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:06.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:06.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:06.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:06.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:06.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:06.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:06.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:06.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:06.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:06.840 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:31:07.599 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-05-25T06:31:07.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:07 smithi179 ceph-mon[67580]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 816 B/s rd, 0 op/s 2024-05-25T06:31:07.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:07 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1902790175' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:07 smithi175 ceph-mon[84648]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 816 B/s rd, 0 op/s 2024-05-25T06:31:07.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:07 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1902790175' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:07 smithi175 ceph-mon[89733]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 816 B/s rd, 0 op/s 2024-05-25T06:31:07.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:07 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1902790175' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:08 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:08.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:08 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:08 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:09.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:09 smithi175 ceph-mon[84648]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-05-25T06:31:09.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:09 smithi175 ceph-mon[89733]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-05-25T06:31:09.687 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)" 2024-05-25T06:31:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:09 smithi179 ceph-mon[67580]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-05-25T06:31:10.404 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:31:10.712 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:10 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1555498388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:10.712 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:10 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1555498388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:10 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1555498388' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:11.805 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:11 smithi175 ceph-mon[84648]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:11.806 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:11 smithi175 ceph-mon[89733]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:12.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:11 smithi179 ceph-mon[67580]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": null, 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": false, 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "", 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": null, 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "", 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:31:12.502 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:31:12.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:12 smithi175 ceph-mon[89733]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:31:12.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:12 smithi175 ceph-mon[89733]: from='client.44604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:12 smithi175 ceph-mon[84648]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:31:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:12 smithi175 ceph-mon[84648]: from='client.44604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:12 smithi179 ceph-mon[67580]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:31:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:12 smithi179 ceph-mon[67580]: from='client.44604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:13.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:31:13.660 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:31:13] "GET /metrics HTTP/1.1" 200 38372 "" "Prometheus/2.43.0" 2024-05-25T06:31:14.649 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:14 smithi175 ceph-mon[89733]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:14.649 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:14 smithi175 ceph-mon[84648]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:14 smithi179 ceph-mon[67580]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:15.265 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:31:15.265 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:31:15.265 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:31:15.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:15 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/357061620' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:31:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:15 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/357061620' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:31:15.963 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-05-25T06:31:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:15 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/357061620' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:31:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:16 smithi175 ceph-mon[84648]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:16 smithi175 ceph-mon[89733]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:17.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:16 smithi179 ceph-mon[67580]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:17.891 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:17 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:17.891 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:17 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:17.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:17 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:18 smithi179 ceph-mon[67580]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:19.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:18 smithi179 ceph-mon[67580]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:19.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:18 smithi175 ceph-mon[84648]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:19.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:18 smithi175 ceph-mon[84648]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:18 smithi175 ceph-mon[89733]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:19.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:18 smithi175 ceph-mon[89733]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:20.304 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:31:21.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-25T06:31:21.562 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:21.562 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:21.563 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:21.564 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:21.564 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:21.564 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:31:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:31:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:21.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[84648]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 876 B/s rd, 0 op/s 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[84648]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[89733]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 876 B/s rd, 0 op/s 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[89733]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:22.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:22 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:22 smithi179 ceph-mon[67580]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 876 B/s rd, 0 op/s 2024-05-25T06:31:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:22 smithi179 ceph-mon[67580]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:22 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:22 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:22 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:23.183 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:31:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:31:23] "GET /metrics HTTP/1.1" 200 38375 "" "Prometheus/2.43.0" 2024-05-25T06:31:23.868 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:31:23.868 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (9m) 2m ago 31m 28.6M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:31:23.868 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (9m) 32s ago 31m 82.5M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:31:23.868 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 2m ago 30m 62.7M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:31:23.868 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (10m) 32s ago 36m 454M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:31:23.868 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (20m) 2m ago 37m 513M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (6m) 2m ago 37m 53.4M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (7m) 32s ago 36m 48.0M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (6m) 2m ago 36m 44.6M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (10m) 2m ago 32m 16.1M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (10m) 32s ago 32m 14.8M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (4m) 2m ago 35m 53.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (4m) 2m ago 35m 51.4M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (3m) 2m ago 34m 41.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (2m) 2m ago 34m 14.9M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (118s) 32s ago 34m 47.5M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (96s) 32s ago 33m 45.0M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (74s) 32s ago 33m 43.6M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 8f5beea10bda 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (37s) 32s ago 33m 12.6M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 771fbf01687f 2024-05-25T06:31:23.869 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (9m) 32s ago 32m 67.1M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:31:23.870 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (30m) 2m ago 30m 78.3M - 17.2.0 e1d6a67b021e ad679e7c248c 2024-05-25T06:31:23.870 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (30m) 32s ago 30m 82.1M - 17.2.0 e1d6a67b021e a8b4b67e16f0 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:31:24.294 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 8 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 13 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:31:24.295 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:31:24.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[89733]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:24.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[89733]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:24.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[89733]: from='client.44628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:24.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1116840723' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:24.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[84648]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:24.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[84648]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:24.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[84648]: from='client.44628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:24.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:24 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1116840723' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:31:24.124+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:24.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:24.572 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:31:24.573 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:24.574 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:24.575 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:24 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "", 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 image", 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:31:24.662 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:31:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:24 smithi179 ceph-mon[67580]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:24 smithi179 ceph-mon[67580]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:24 smithi179 ceph-mon[67580]: from='client.44628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:24 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1116840723' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:25.626 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:25 smithi175 ceph-mon[89733]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:25.626 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:25 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:25.626 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:25.626 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:25.626 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:25.627 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:25.628 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:25.629 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:25 smithi175 ceph-mon[84648]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:25 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:25.630 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:25.631 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:25.632 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:25.633 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:25.634 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:25 smithi175 ceph-mon[84648]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.635 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:25 smithi175 ceph-mon[89733]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:31:25.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:25 smithi179 ceph-mon[67580]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:25.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:25 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.786 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:25.787 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:25.788 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:25.789 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:25.790 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:25.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:25 smithi179 ceph-mon[67580]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:31:26.643 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:26 smithi175 ceph-mon[89733]: from='client.54583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:26.644 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:26 smithi175 ceph-mon[89733]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:26.644 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:26 smithi175 ceph-mon[84648]: from='client.54583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:26.644 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:26 smithi175 ceph-mon[84648]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:26 smithi179 ceph-mon[67580]: from='client.54583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:26 smithi179 ceph-mon[67580]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:31:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:31:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mgr 2024-05-25T06:31:27.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mon 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all crash 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all osd 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mds 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:27.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[89733]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mgr 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mon 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all crash 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all osd 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mds 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:27.823 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:27 smithi175 ceph-mon[84648]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:31:28.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mgr 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mon 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all crash 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:28.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:28.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all osd 2024-05-25T06:31:28.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:28.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:28.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mds 2024-05-25T06:31:28.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:28.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:28.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:27 smithi179 ceph-mon[67580]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:29 smithi179 ceph-mon[67580]: Upgrade: Updating rgw.foo.smithi175.iwhmky (1/2) 2024-05-25T06:31:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:29 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:29 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:31:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:29 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:29 smithi179 ceph-mon[67580]: Deploying daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[84648]: Upgrade: Updating rgw.foo.smithi175.iwhmky (1/2) 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[84648]: Deploying daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[89733]: Upgrade: Updating rgw.foo.smithi175.iwhmky (1/2) 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi175.iwhmky", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:29 smithi175 ceph-mon[89733]: Deploying daemon rgw.foo.smithi175.iwhmky on smithi175 2024-05-25T06:31:30.473 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:30 smithi175 ceph-mon[89733]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:30.473 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:30 smithi175 ceph-mon[84648]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:30 smithi179 ceph-mon[67580]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-25T06:31:32.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:32 smithi175 ceph-mon[89733]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:31:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:32 smithi175 ceph-mon[84648]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:31:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:32 smithi179 ceph-mon[67580]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:31:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:33.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:33.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:33.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:33.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:33.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:31:33] "GET /metrics HTTP/1.1" 200 38375 "" "Prometheus/2.43.0" 2024-05-25T06:31:34.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:34 smithi179 ceph-mon[67580]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:34.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:34.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:31:34.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:34 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[84648]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:31:34.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[89733]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:31:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:34.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi179.hopgfy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-25T06:31:34.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:34 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:35.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:35 smithi179 ceph-mon[67580]: Upgrade: Updating rgw.foo.smithi179.hopgfy (2/2) 2024-05-25T06:31:35.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:35 smithi179 ceph-mon[67580]: Deploying daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:31:35.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:35 smithi175 ceph-mon[84648]: Upgrade: Updating rgw.foo.smithi179.hopgfy (2/2) 2024-05-25T06:31:35.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:35 smithi175 ceph-mon[84648]: Deploying daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:31:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:35 smithi175 ceph-mon[89733]: Upgrade: Updating rgw.foo.smithi179.hopgfy (2/2) 2024-05-25T06:31:35.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:35 smithi175 ceph-mon[89733]: Deploying daemon rgw.foo.smithi179.hopgfy on smithi179 2024-05-25T06:31:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:36 smithi179 ceph-mon[67580]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 85 B/s wr, 33 op/s 2024-05-25T06:31:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:36.789 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:36 smithi175 ceph-mon[84648]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 85 B/s wr, 33 op/s 2024-05-25T06:31:36.790 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:36.790 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:36 smithi175 ceph-mon[89733]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 85 B/s wr, 33 op/s 2024-05-25T06:31:36.790 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:38 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:38 smithi179 ceph-mon[67580]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-05-25T06:31:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:38 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:38.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:38 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:38.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:38 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[84648]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-05-25T06:31:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[89733]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-05-25T06:31:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:38.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:38 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:40.755 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:40 smithi179 ceph-mon[67580]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 170 B/s wr, 112 op/s 2024-05-25T06:31:40.774 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:40 smithi175 ceph-mon[84648]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 170 B/s wr, 112 op/s 2024-05-25T06:31:40.775 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:40 smithi175 ceph-mon[89733]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 170 B/s wr, 112 op/s 2024-05-25T06:31:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:42 smithi179 ceph-mon[67580]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 170 B/s wr, 142 op/s 2024-05-25T06:31:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:42 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:42 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:42 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:42 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[84648]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 170 B/s wr, 142 op/s 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[89733]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 170 B/s wr, 142 op/s 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:42 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:31:43] "GET /metrics HTTP/1.1" 200 38370 "" "Prometheus/2.43.0" 2024-05-25T06:31:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:43 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:43 smithi179 ceph-mon[67580]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-05-25T06:31:44.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[89733]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:44.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:43 smithi175 ceph-mon[84648]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-05-25T06:31:46.474 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:46 smithi175 ceph-mon[89733]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-05-25T06:31:46.474 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:46 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:46.474 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:46 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:46.475 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:46 smithi175 ceph-mon[84648]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-05-25T06:31:46.475 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:46 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:46.475 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:46 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:46 smithi179 ceph-mon[67580]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-05-25T06:31:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:46 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:46 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:47.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 90 B/s wr, 144 op/s 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 110 B/s wr, 109 op/s 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:47.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 90 B/s wr, 144 op/s 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 110 B/s wr, 109 op/s 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:47.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:47.822 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:47 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 90 B/s wr, 144 op/s 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 110 B/s wr, 109 op/s 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:48.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:48.037 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:47 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:49.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:31:48.606+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:49.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:49.070 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:49.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:49.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:49.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:49.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:49.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:49.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:49.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:49.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:49.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:49.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:49.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:49.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:48 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:49.919 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:49.920 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:49.921 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:49.922 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:49.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:49.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:49.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:49.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:49.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:49.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:49.923 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:49.924 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:49.925 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:49.926 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:49.927 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all rgw 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi175.iwhmky"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi175.iwhmky"}]': finished 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi179.hopgfy"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi179.hopgfy"}]': finished 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.929 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.930 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all rgw 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi175.iwhmky"}]: dispatch 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi175.iwhmky"}]': finished 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi179.hopgfy"}]: dispatch 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi179.hopgfy"}]': finished 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.931 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:31:49.932 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.932 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.932 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.932 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:31:49.932 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nfs 2024-05-25T06:31:49.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:31:49.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.933 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.934 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.934 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.934 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.934 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.934 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.935 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Upgrade: Finalizing container_image settings 2024-05-25T06:31:49.935 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:31:49.935 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:31:49.935 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.935 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:31:49.935 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:31:49.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:31:49.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:31:49.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-25T06:31:49.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:31:49.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:31:49.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:31:49.936 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.937 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.938 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.938 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.938 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.938 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.938 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Upgrade: Complete! 2024-05-25T06:31:49.938 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 40 op/s 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Checking dashboard <-> RGW credentials 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:49.939 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nfs 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.940 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Upgrade: Finalizing container_image settings 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-25T06:31:49.941 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:31:49.942 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Upgrade: Complete! 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 40 op/s 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-05-25T06:31:49.943 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Checking dashboard <-> RGW credentials 2024-05-25T06:31:49.944 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:49.944 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:49.944 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:49.944 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:49.944 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:49 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:50.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:50.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:50.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:50.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all rgw 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi175.iwhmky"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi175.iwhmky"}]': finished 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi179.hopgfy"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi179.hopgfy"}]': finished 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.041 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nfs 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all nvmeof 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:31:50.042 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Upgrade: Finalizing container_image settings 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-25T06:31:50.043 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-25T06:31:50.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-25T06:31:50.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-25T06:31:50.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-25T06:31:50.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-25T06:31:50.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Upgrade: Complete! 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:50.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 40 op/s 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Checking dashboard <-> RGW credentials 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:50.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:49 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:50 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:51.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:50 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:50 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:51.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:50 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:51.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:50 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:31:51.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:50 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:51.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:31:50.838+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:51.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:51.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:51.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:51.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:51.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:51.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:51.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:51.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:51.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:51.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:51.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:51.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:51.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:51.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:51.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:51.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:51.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.076 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:51.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:51.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:51.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:51.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:51.077 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:51.078 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:51.078 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:51.078 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:31:51.078 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:51.078 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:31:51.078 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:51.079 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:31:51.079 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:51.079 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:31:51.079 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:51.079 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:51.079 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:51.079 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:51.080 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:51.080 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:51.080 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:51.080 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.080 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.080 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:51.081 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:51.081 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:51.081 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:51.081 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:51.081 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:51.082 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:51.082 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:51.082 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:51.082 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.083 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:50 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.918 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:51.919 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:51.920 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:51.921 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:51.922 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.923 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.924 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:51.925 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:51.926 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:51.927 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:51.928 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:51.929 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:51 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-25T06:31:52.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:52.036 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:52.037 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:52.038 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:52.039 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:31:52.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:51 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:31:52.755+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:53.071 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:53.072 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:53.073 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:53.074 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:53.075 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:53.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:31:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:31:53] "GET /metrics HTTP/1.1" 200 38407 "" "Prometheus/2.43.0" 2024-05-25T06:31:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:53 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:54.285 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:54.286 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:31:54.287 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.288 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:53 smithi179 ceph-mon[67580]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:53 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:54.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:54.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:54.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:54.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:53 smithi175 ceph-mon[84648]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:53 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:54.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:54.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:31:54.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:31:54.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:31:54.328 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:53 smithi175 ceph-mon[89733]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s 2024-05-25T06:31:55.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:31:56.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:55 smithi175 ceph-mon[84648]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 5 op/s 2024-05-25T06:31:56.027 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:55 smithi175 ceph-mon[84648]: from='client.54664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:56.027 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:55 smithi175 ceph-mon[89733]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 5 op/s 2024-05-25T06:31:56.027 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:55 smithi175 ceph-mon[89733]: from='client.54664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:56.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:55 smithi179 ceph-mon[67580]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 5 op/s 2024-05-25T06:31:56.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:55 smithi179 ceph-mon[67580]: from='client.54664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:57.850 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (10m) 15s ago 32m 28.2M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (9m) 16s ago 31m 82.1M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 15s ago 31m 62.9M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (11m) 16s ago 36m 454M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (21m) 15s ago 38m 518M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (7m) 15s ago 38m 60.5M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (8m) 16s ago 36m 51.2M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (7m) 15s ago 37m 50.9M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (10m) 15s ago 32m 16.0M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:31:57.851 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (10m) 16s ago 32m 15.0M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (5m) 15s ago 36m 62.0M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (4m) 15s ago 35m 59.6M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (3m) 15s ago 35m 47.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (2m) 15s ago 35m 61.1M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (2m) 16s ago 34m 52.2M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (2m) 16s ago 34m 48.1M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (107s) 16s ago 34m 46.5M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 8f5beea10bda 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (71s) 16s ago 33m 47.9M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 771fbf01687f 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (10m) 16s ago 32m 68.8M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (25s) 15s ago 31m 77.1M - 19.0.0-2843-gea0684c0 f9a9809cafd6 43b35e3d1335 2024-05-25T06:31:57.852 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (20s) 16s ago 31m 74.1M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a799d7d558b1 2024-05-25T06:31:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:57 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:58.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:57 smithi179 ceph-mon[67580]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-05-25T06:31:58.215 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:57 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:58.216 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:57 smithi175 ceph-mon[89733]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-05-25T06:31:58.216 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:57 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:31:58.216 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:57 smithi175 ceph-mon[84648]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-05-25T06:31:58.586 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-05-25T06:31:59.153 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:58 smithi175 ceph-mon[89733]: from='client.54670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:59.153 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:58 smithi175 ceph-mon[84648]: from='client.54670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:31:59.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:58 smithi179 ceph-mon[67580]: from='client.54670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:00.251 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:31:59 smithi175 ceph-mon[89733]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-25T06:32:00.252 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:31:59 smithi175 ceph-mon[84648]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-25T06:32:00.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:31:59 smithi179 ceph-mon[67580]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-25T06:32:00.665 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:32:01.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:00 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1282313353' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:01.319 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:00 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1282313353' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:01.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:00 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1282313353' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:01.492 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-05-25T06:32:02.065 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:01 smithi175 ceph-mon[89733]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-25T06:32:02.066 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:01 smithi175 ceph-mon[84648]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-25T06:32:02.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:01 smithi179 ceph-mon[67580]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-25T06:32:03.647 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)" 2024-05-25T06:32:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:32:03] "GET /metrics HTTP/1.1" 200 38407 "" "Prometheus/2.43.0" 2024-05-25T06:32:04.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:03 smithi179 ceph-mon[67580]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-25T06:32:04.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:03 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/180818122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:04.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:03 smithi175 ceph-mon[89733]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-25T06:32:04.319 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:03 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/180818122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:04.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:03 smithi175 ceph-mon[84648]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-25T06:32:04.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:03 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/180818122' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:04.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:32:06.240 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:05 smithi175 ceph-mon[89733]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-25T06:32:06.240 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:06.240 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:05 smithi175 ceph-mon[84648]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-25T06:32:06.240 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:06.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:05 smithi179 ceph-mon[67580]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-25T06:32:06.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:06.469 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": null, 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": false, 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "", 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": null, 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "", 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:32:06.470 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:32:07.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:06 smithi175 ceph-mon[89733]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:07.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:06 smithi175 ceph-mon[84648]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:07.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:32:07.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:06 smithi179 ceph-mon[67580]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:08.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:07 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:08.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:07 smithi175 ceph-mon[84648]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:08.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:07 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:08.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:07 smithi175 ceph-mon[89733]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:08.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:07 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:08.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:07 smithi179 ceph-mon[67580]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:09.214 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:32:09.214 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:32:09.214 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:32:09.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-25T06:32:10.161 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:09 smithi175 ceph-mon[89733]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:10.162 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:09 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/176531318' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:32:10.162 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:09 smithi175 ceph-mon[84648]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:10.162 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:09 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/176531318' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:32:10.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:09 smithi179 ceph-mon[67580]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:10.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:09 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/176531318' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:32:12.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:11 smithi175 ceph-mon[89733]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:12.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:11 smithi175 ceph-mon[84648]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:12.083 INFO:teuthology.orchestra.run.smithi175.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:32:12.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:11 smithi179 ceph-mon[67580]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:12.904 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-25T06:32:12.913 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi175.front.sepia.ceph.com 2024-05-25T06:32:12.913 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='client.54700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:13.133 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='client.54700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:13.134 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-05-25T06:32:13.135 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:13.135 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:13.135 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:32:13.135 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:13.135 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:13.135 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:13 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='client.54700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:32:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:13.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-05-25T06:32:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:32:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:13.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:13 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:13.746 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:32:13] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-05-25T06:32:15.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:32:14.979+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:15.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:32:15.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:15.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:15.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:15.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:15.177 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:15.178 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:15.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:15.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:15.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:32:15.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:32:15.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:32:15.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:32:15.179 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return future.result(timeout) 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return self.__get_result() 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: raise self._exception 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: use_current_daemon_image=reconfig, 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Creating ceph-iscsi config... 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:15.180 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: Traceback (most recent call last): 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: return _run_code(code, main_globals, None, 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: exec(code, run_globals) 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:15.181 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:15.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:15.182 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:14 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:15.498 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:15 smithi175 ceph-mon[84648]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:15.499 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:15 smithi175 ceph-mon[89733]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:15.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:15 smithi179 ceph-mon[67580]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:16.116 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:16 smithi175 ceph-mon[84648]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.320 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.321 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return future.result(timeout) 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: return self.__get_result() 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: raise self._exception 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:32:16.322 INFO:journalctl@ceph.mon.a.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:16.323 INFO:journalctl@ceph.mon.a.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:16 smithi175 ceph-mon[84648]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:16 smithi175 ceph-mon[89733]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:16.324 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:16.325 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:32:16.326 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return future.result(timeout) 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: return self.__get_result() 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: raise self._exception 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Creating ceph-iscsi config... 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:16.327 INFO:journalctl@ceph.mon.c.smithi175.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:16.328 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:16.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:16.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:16.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.329 INFO:journalctl@ceph.mon.c.smithi175.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.329 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:16 smithi175 ceph-mon[89733]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:32:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:16 smithi179 ceph-mon[67580]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:32:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:16.535 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:16.536 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: return future.result(timeout) 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.__get_result() 2024-05-25T06:32:16.537 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: raise self._exception 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_current_daemon_image=reconfig, 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Creating ceph-iscsi config... 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Non-zero exit code 1 from systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.538 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: exec(code, run_globals) 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1123, in deploy_daemon 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: RuntimeError: Failed command: systemctl restart ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout: See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:32:16.539 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:16 smithi179 ceph-mon[67580]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 2024-05-25T06:32:16.816 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:32:16.816 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (10m) 34s ago 32m 28.2M - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:32:16.816 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (9m) 34s ago 32m 82.1M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:32:16.816 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 unknown 34s ago 31m 62.9M - 3.5 e1d6a67b021e 6f7cb8a21130 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (11m) 35s ago 36m 454M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (21m) 34s ago 38m 518M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (7m) 34s ago 38m 60.5M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (8m) 35s ago 37m 51.2M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (7m) 34s ago 37m 50.9M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (11m) 34s ago 33m 16.0M - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (11m) 34s ago 33m 15.0M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (5m) 34s ago 36m 62.0M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (5m) 34s ago 36m 59.6M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (4m) 34s ago 35m 47.8M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (3m) 34s ago 35m 61.1M 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (2m) 35s ago 35m 52.2M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:32:16.817 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (2m) 35s ago 34m 48.1M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:32:16.818 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (2m) 34s ago 34m 46.5M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 8f5beea10bda 2024-05-25T06:32:16.818 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (90s) 34s ago 33m 47.9M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 771fbf01687f 2024-05-25T06:32:16.818 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (10m) 34s ago 33m 68.8M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:32:16.818 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (44s) 34s ago 31m 77.1M - 19.0.0-2843-gea0684c0 f9a9809cafd6 43b35e3d1335 2024-05-25T06:32:16.818 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (39s) 34s ago 31m 74.1M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a799d7d558b1 2024-05-25T06:32:17.247 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 8 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:17.248 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:32:17.249 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 15 2024-05-25T06:32:17.249 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:32:17.249 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:32:17.436 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:17 smithi179 ceph-mon[67580]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:17.436 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:17 smithi179 ceph-mon[67580]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:17.437 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:17 smithi179 ceph-mon[67580]: from='client.44772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:17.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:17 smithi175 ceph-mon[84648]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:17.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:17 smithi175 ceph-mon[84648]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:17.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:17 smithi175 ceph-mon[84648]: from='client.44772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:17 smithi175 ceph-mon[89733]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:17 smithi175 ceph-mon[89733]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:17.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:17 smithi175 ceph-mon[89733]: from='client.44772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:17.587 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:32:17.587 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78", 2024-05-25T06:32:17.587 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:32:17.588 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-25T06:32:17.588 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [], 2024-05-25T06:32:17.588 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "", 2024-05-25T06:32:17.588 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ea0684c08cbcf83f4af9930de2267b3563ac5f78 image", 2024-05-25T06:32:17.588 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": false 2024-05-25T06:32:17.588 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:32:18.005 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-25T06:32:18.006 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-25T06:32:18.006 INFO:teuthology.orchestra.run.smithi175.stdout: daemon iscsi.foo.smithi175.buback on smithi175 is in unknown state 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1415432383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.320 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.321 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3861784179' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1415432383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.322 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.323 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:18 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3861784179' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:32:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1415432383' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:18.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:18.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:18 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3861784179' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:32:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:32:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:32:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mgr 2024-05-25T06:32:19.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mon 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all crash 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all osd 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all mds 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all rgw 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:32:19.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:19 smithi179 ceph-mon[67580]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mgr 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mon 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all crash 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all osd 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all mds 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all rgw 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:32:19.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[89733]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Target is version 19.0.0-2843-gea0684c0 (squid) 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, digests ['quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee'] 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mgr 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mon 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all crash 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all osd 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all mds 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all rgw 2024-05-25T06:32:19.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all rbd-mirror 2024-05-25T06:32:19.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-25T06:32:19.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: Upgrade: Setting container_image for all ceph-exporter 2024-05-25T06:32:19.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:19 smithi175 ceph-mon[84648]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:20 smithi179 ceph-mon[67580]: Upgrade: Updating iscsi.foo.smithi175.buback 2024-05-25T06:32:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:20 smithi179 ceph-mon[67580]: Deploying daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:20 smithi179 ceph-mon[67580]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:21.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:20 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[84648]: Upgrade: Updating iscsi.foo.smithi175.buback 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[84648]: Deploying daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[84648]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:21.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[89733]: Upgrade: Updating iscsi.foo.smithi175.buback 2024-05-25T06:32:21.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:21.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:21.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:21.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[89733]: Deploying daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:21.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[89733]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:21.071 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:20 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:21.535 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:32:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=cleanup t=2024-05-25T06:32:21.161253708Z level=info msg="Completed cleanup jobs" duration=2.378229ms 2024-05-25T06:32:23.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:23 smithi179 ceph-mon[67580]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:32:23.555 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:23 smithi175 ceph-mon[84648]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:32:23.555 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:23 smithi175 ceph-mon[89733]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:32:23.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:32:23] "GET /metrics HTTP/1.1" 200 38404 "" "Prometheus/2.43.0" 2024-05-25T06:32:25.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:25 smithi179 ceph-mon[67580]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:25.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:25 smithi175 ceph-mon[84648]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:25.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:25 smithi175 ceph-mon[89733]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:27.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:27 smithi179 ceph-mon[67580]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:27.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:27 smithi175 ceph-mon[84648]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:27.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:27 smithi175 ceph-mon[89733]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:28.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:28 smithi179 ceph-mon[67580]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:28.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:28 smithi175 ceph-mon[84648]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:28.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:28 smithi175 ceph-mon[89733]: from='client.25213 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:29.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:29 smithi179 ceph-mon[67580]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:29.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:29 smithi175 ceph-mon[84648]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:29.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:29 smithi175 ceph-mon[89733]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:31.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:31 smithi179 ceph-mon[67580]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:31.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:31 smithi175 ceph-mon[84648]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:31.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:31 smithi175 ceph-mon[89733]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:33.069 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:32 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:32:32.633+0000 7f7ea892e700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. 2024-05-25T06:32:33.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:33.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:33.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:33.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:33.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:33.339 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:33.340 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:33 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:33.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:33.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:33.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:33.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi175.buback", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-25T06:32:33.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:33 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:33.591 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:32:33] "GET /metrics HTTP/1.1" 200 38404 "" "Prometheus/2.43.0" 2024-05-25T06:32:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:34 smithi179 ceph-mon[67580]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. 2024-05-25T06:32:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:34 smithi179 ceph-mon[67580]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-05-25T06:32:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:34 smithi179 ceph-mon[67580]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-05-25T06:32:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:34 smithi179 ceph-mon[67580]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 785 B/s rd, 0 op/s 2024-05-25T06:32:34.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:34 smithi179 ceph-mon[67580]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:32:34.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:34 smithi179 ceph-mon[67580]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:34.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:34 smithi179 ceph-mon[67580]: Health check failed: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-25T06:32:34.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[84648]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. 2024-05-25T06:32:34.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[84648]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[84648]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[84648]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 785 B/s rd, 0 op/s 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[84648]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[84648]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[84648]: Health check failed: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[89733]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[89733]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[89733]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-05-25T06:32:34.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[89733]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 785 B/s rd, 0 op/s 2024-05-25T06:32:34.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[89733]: Reconfiguring iscsi.foo.smithi175.buback (dependencies changed)... 2024-05-25T06:32:34.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[89733]: Reconfiguring daemon iscsi.foo.smithi175.buback on smithi175 2024-05-25T06:32:34.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:34 smithi175 ceph-mon[89733]: Health check failed: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-25T06:32:36.516 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:36.516 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/408898807' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3941285676' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4249004229"}]: dispatch 2024-05-25T06:32:36.517 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/408898807' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:32:36.518 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:36 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3941285676' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4249004229"}]: dispatch 2024-05-25T06:32:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:32:36.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-25T06:32:36.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:32:36.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-25T06:32:36.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:32:36.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:36.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:36.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/408898807' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-25T06:32:36.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:36 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3941285676' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4249004229"}]: dispatch 2024-05-25T06:32:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:37 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3941285676' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4249004229"}]': finished 2024-05-25T06:32:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:37 smithi179 ceph-mon[67580]: osdmap e150: 8 total, 8 up, 8 in 2024-05-25T06:32:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:37 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3908118443' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]: dispatch 2024-05-25T06:32:37.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:37 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]: dispatch 2024-05-25T06:32:37.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3941285676' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4249004229"}]': finished 2024-05-25T06:32:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[84648]: osdmap e150: 8 total, 8 up, 8 in 2024-05-25T06:32:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3908118443' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]: dispatch 2024-05-25T06:32:37.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]: dispatch 2024-05-25T06:32:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3941285676' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/4249004229"}]': finished 2024-05-25T06:32:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[89733]: osdmap e150: 8 total, 8 up, 8 in 2024-05-25T06:32:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3908118443' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]: dispatch 2024-05-25T06:32:37.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:37 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]: dispatch 2024-05-25T06:32:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:38 smithi179 ceph-mon[67580]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 0 op/s 2024-05-25T06:32:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:38 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]': finished 2024-05-25T06:32:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:38 smithi179 ceph-mon[67580]: osdmap e151: 8 total, 8 up, 8 in 2024-05-25T06:32:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:38 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/65316249' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]: dispatch 2024-05-25T06:32:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:38 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]: dispatch 2024-05-25T06:32:38.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[84648]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 0 op/s 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]': finished 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[84648]: osdmap e151: 8 total, 8 up, 8 in 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/65316249' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]: dispatch 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]: dispatch 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[89733]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 0 op/s 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2749146691"}]': finished 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[89733]: osdmap e151: 8 total, 8 up, 8 in 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/65316249' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]: dispatch 2024-05-25T06:32:38.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:38 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]: dispatch 2024-05-25T06:32:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]': finished 2024-05-25T06:32:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: osdmap e152: 8 total, 8 up, 8 in 2024-05-25T06:32:39.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1069337553' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1019556141"}]: dispatch 2024-05-25T06:32:39.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:39.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:39.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:39 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]': finished 2024-05-25T06:32:39.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: osdmap e152: 8 total, 8 up, 8 in 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1069337553' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1019556141"}]: dispatch 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/4243093590"}]': finished 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: osdmap e152: 8 total, 8 up, 8 in 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1069337553' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1019556141"}]: dispatch 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:39.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:32:39.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:32:39.821 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:39 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:40 smithi179 ceph-mon[67580]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-25T06:32:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:40 smithi179 ceph-mon[67580]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 1 op/s 2024-05-25T06:32:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:40 smithi179 ceph-mon[67580]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:32:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:40 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1069337553' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1019556141"}]': finished 2024-05-25T06:32:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:40 smithi179 ceph-mon[67580]: osdmap e153: 8 total, 8 up, 8 in 2024-05-25T06:32:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:40 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/4091199353' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3153018563"}]: dispatch 2024-05-25T06:32:40.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[89733]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[89733]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 1 op/s 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[89733]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1069337553' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1019556141"}]': finished 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[89733]: osdmap e153: 8 total, 8 up, 8 in 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/4091199353' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3153018563"}]: dispatch 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[84648]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[84648]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 1 op/s 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[84648]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1069337553' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/1019556141"}]': finished 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[84648]: osdmap e153: 8 total, 8 up, 8 in 2024-05-25T06:32:40.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:40 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/4091199353' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3153018563"}]: dispatch 2024-05-25T06:32:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:41 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/4091199353' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3153018563"}]': finished 2024-05-25T06:32:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:41 smithi179 ceph-mon[67580]: osdmap e154: 8 total, 8 up, 8 in 2024-05-25T06:32:41.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:41 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2474125859' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/4243093590"}]: dispatch 2024-05-25T06:32:41.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:41 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/4091199353' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3153018563"}]': finished 2024-05-25T06:32:41.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:41 smithi175 ceph-mon[89733]: osdmap e154: 8 total, 8 up, 8 in 2024-05-25T06:32:41.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:41 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2474125859' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/4243093590"}]: dispatch 2024-05-25T06:32:41.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:41 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/4091199353' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3153018563"}]': finished 2024-05-25T06:32:41.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:41 smithi175 ceph-mon[84648]: osdmap e154: 8 total, 8 up, 8 in 2024-05-25T06:32:41.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:41 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2474125859' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/4243093590"}]: dispatch 2024-05-25T06:32:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:42 smithi179 ceph-mon[67580]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:42 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2474125859' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/4243093590"}]': finished 2024-05-25T06:32:42.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:42 smithi179 ceph-mon[67580]: osdmap e155: 8 total, 8 up, 8 in 2024-05-25T06:32:42.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:42 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2561773356' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]: dispatch 2024-05-25T06:32:42.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:42 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]: dispatch 2024-05-25T06:32:42.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[89733]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2474125859' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/4243093590"}]': finished 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[89733]: osdmap e155: 8 total, 8 up, 8 in 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2561773356' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]: dispatch 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]: dispatch 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[84648]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2474125859' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/4243093590"}]': finished 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[84648]: osdmap e155: 8 total, 8 up, 8 in 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2561773356' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]: dispatch 2024-05-25T06:32:42.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:42 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]: dispatch 2024-05-25T06:32:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:32:43] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-05-25T06:32:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]': finished 2024-05-25T06:32:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[84648]: osdmap e156: 8 total, 8 up, 8 in 2024-05-25T06:32:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1495845654' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]: dispatch 2024-05-25T06:32:43.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]: dispatch 2024-05-25T06:32:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]': finished 2024-05-25T06:32:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[89733]: osdmap e156: 8 total, 8 up, 8 in 2024-05-25T06:32:43.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1495845654' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]: dispatch 2024-05-25T06:32:43.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:43 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]: dispatch 2024-05-25T06:32:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:43 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/272542050"}]': finished 2024-05-25T06:32:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:43 smithi179 ceph-mon[67580]: osdmap e156: 8 total, 8 up, 8 in 2024-05-25T06:32:44.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:43 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1495845654' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]: dispatch 2024-05-25T06:32:44.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:43 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]: dispatch 2024-05-25T06:32:44.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[89733]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:32:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]': finished 2024-05-25T06:32:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[89733]: osdmap e157: 8 total, 8 up, 8 in 2024-05-25T06:32:44.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3704780471' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3499532044"}]: dispatch 2024-05-25T06:32:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[84648]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:32:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]': finished 2024-05-25T06:32:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[84648]: osdmap e157: 8 total, 8 up, 8 in 2024-05-25T06:32:44.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:44 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3704780471' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3499532044"}]: dispatch 2024-05-25T06:32:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:44 smithi179 ceph-mon[67580]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail 2024-05-25T06:32:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:44 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2905170780"}]': finished 2024-05-25T06:32:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:44 smithi179 ceph-mon[67580]: osdmap e157: 8 total, 8 up, 8 in 2024-05-25T06:32:45.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:44 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3704780471' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3499532044"}]: dispatch 2024-05-25T06:32:45.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3704780471' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3499532044"}]': finished 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[84648]: osdmap e158: 8 total, 8 up, 8 in 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/432708805' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2049831035"}]: dispatch 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[84648]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3704780471' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3499532044"}]': finished 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[89733]: osdmap e158: 8 total, 8 up, 8 in 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/432708805' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2049831035"}]: dispatch 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[89733]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:45.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:45 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:45 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3704780471' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/3499532044"}]': finished 2024-05-25T06:32:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:45 smithi179 ceph-mon[67580]: osdmap e158: 8 total, 8 up, 8 in 2024-05-25T06:32:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:45 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/432708805' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2049831035"}]: dispatch 2024-05-25T06:32:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:45 smithi179 ceph-mon[67580]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:46.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:45 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:46.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/432708805' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2049831035"}]': finished 2024-05-25T06:32:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[89733]: osdmap e159: 8 total, 8 up, 8 in 2024-05-25T06:32:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3210656908' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]: dispatch 2024-05-25T06:32:46.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]: dispatch 2024-05-25T06:32:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/432708805' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2049831035"}]': finished 2024-05-25T06:32:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[84648]: osdmap e159: 8 total, 8 up, 8 in 2024-05-25T06:32:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3210656908' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]: dispatch 2024-05-25T06:32:46.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:46 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]: dispatch 2024-05-25T06:32:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:46 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/432708805' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6800/2049831035"}]': finished 2024-05-25T06:32:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:46 smithi179 ceph-mon[67580]: osdmap e159: 8 total, 8 up, 8 in 2024-05-25T06:32:47.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:46 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3210656908' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]: dispatch 2024-05-25T06:32:47.036 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:46 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]: dispatch 2024-05-25T06:32:47.936 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:47 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]': finished 2024-05-25T06:32:47.937 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:47 smithi179 ceph-mon[67580]: osdmap e160: 8 total, 8 up, 8 in 2024-05-25T06:32:47.937 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:47 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/873692211' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]: dispatch 2024-05-25T06:32:47.937 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:47 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]: dispatch 2024-05-25T06:32:47.937 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:47 smithi179 ceph-mon[67580]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:48.069 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]': finished 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[89733]: osdmap e160: 8 total, 8 up, 8 in 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/873692211' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]: dispatch 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]: dispatch 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[89733]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:6801/2049831035"}]': finished 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[84648]: osdmap e160: 8 total, 8 up, 8 in 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/873692211' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]: dispatch 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]: dispatch 2024-05-25T06:32:48.070 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:47 smithi175 ceph-mon[84648]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:48.364 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:32:48.701 INFO:teuthology.orchestra.run.smithi175.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed." 2024-05-25T06:32:48.701 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:48 smithi175 ceph-mon[89733]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]': finished 2024-05-25T06:32:48.701 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:48 smithi175 ceph-mon[89733]: osdmap e161: 8 total, 8 up, 8 in 2024-05-25T06:32:48.701 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:48 smithi175 ceph-mon[89733]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:48.702 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:48 smithi175 ceph-mon[84648]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]': finished 2024-05-25T06:32:48.702 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:48 smithi175 ceph-mon[84648]: osdmap e161: 8 total, 8 up, 8 in 2024-05-25T06:32:48.702 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:48 smithi175 ceph-mon[84648]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:48 smithi179 ceph-mon[67580]: from='client.? ' entity='client.iscsi.foo.smithi175.buback' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.175:0/2379021700"}]': finished 2024-05-25T06:32:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:48 smithi179 ceph-mon[67580]: osdmap e161: 8 total, 8 up, 8 in 2024-05-25T06:32:49.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:48 smithi179 ceph-mon[67580]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:49.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:32:49.973 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:49 smithi175 ceph-mon[84648]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:49.974 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:49 smithi175 ceph-mon[84648]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:49.974 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:49 smithi175 ceph-mon[89733]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:49.974 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:49 smithi175 ceph-mon[89733]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:49 smithi179 ceph-mon[67580]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:50.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:49 smithi179 ceph-mon[67580]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-25T06:32:51.627 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:32:51.627 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (10m) 12s ago 33m - - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:32:51.627 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (10m) 69s ago 32m 82.1M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:32:51.627 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (16s) 12s ago 32m - - 3.8 f9a9809cafd6 64013f30a147 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (12m) 69s ago 37m 454M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (22m) 12s ago 39m - - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (8m) 12s ago 39m - 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (9m) 69s ago 37m 51.2M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (7m) 12s ago 37m - 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (11m) 12s ago 33m - - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (11m) 69s ago 33m 15.0M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (6m) 12s ago 37m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (5m) 12s ago 36m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (4m) 12s ago 36m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (3m) 12s ago 36m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (3m) 69s ago 35m 52.2M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (3m) 69s ago 35m 48.1M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (2m) 69s ago 34m 46.5M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 8f5beea10bda 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (2m) 69s ago 34m 47.9M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 771fbf01687f 2024-05-25T06:32:51.628 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (11m) 69s ago 33m 68.8M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:32:51.629 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (79s) 12s ago 32m - - 19.0.0-2843-gea0684c0 f9a9809cafd6 43b35e3d1335 2024-05-25T06:32:51.629 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (74s) 69s ago 32m 74.1M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a799d7d558b1 2024-05-25T06:32:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:51.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:51.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:32:51.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:32:52.396 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions' 2024-05-25T06:32:52.651 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:52 smithi175 ceph-mon[89733]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:52.652 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:52 smithi175 ceph-mon[84648]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:52 smithi179 ceph-mon[67580]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:32:53.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:53 smithi179 ceph-mon[67580]: from='client.44910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:53.787 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:53 smithi175 ceph-mon[89733]: from='client.44910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:53.788 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:53 smithi175 ceph-mon[84648]: from='client.44910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:32:53.788 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:32:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:32:53] "GET /metrics HTTP/1.1" 200 38483 "" "Prometheus/2.43.0" 2024-05-25T06:32:54.493 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 8 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:32:54.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:32:54.495 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:32:54.495 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 15 2024-05-25T06:32:54.495 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:32:54.495 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:32:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:54 smithi179 ceph-mon[67580]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 669 B/s rd, 0 op/s 2024-05-25T06:32:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:54 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/4067825419' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:54.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:54 smithi175 ceph-mon[84648]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 669 B/s rd, 0 op/s 2024-05-25T06:32:54.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:54 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/4067825419' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:54 smithi175 ceph-mon[89733]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 669 B/s rd, 0 op/s 2024-05-25T06:32:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:54 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/4067825419' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:32:55.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-05-25T06:32:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:56 smithi179 ceph-mon[67580]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:56 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:56.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:56 smithi175 ceph-mon[89733]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:56 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:56 smithi175 ceph-mon[84648]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:32:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:56 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:32:57.248 INFO:teuthology.orchestra.run.smithi175.stdout:wait for servicemap items w/ changing names to refresh 2024-05-25T06:32:57.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'sleep 60' 2024-05-25T06:32:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:32:58 smithi179 ceph-mon[67580]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:32:58.800 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:32:58 smithi175 ceph-mon[89733]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:32:58.800 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:32:58 smithi175 ceph-mon[84648]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-25T06:33:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:00 smithi179 ceph-mon[67580]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:33:00.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:00 smithi175 ceph-mon[84648]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:33:00.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:00 smithi175 ceph-mon[89733]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-25T06:33:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:02 smithi179 ceph-mon[67580]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:02.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:02 smithi175 ceph-mon[84648]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:02.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:02 smithi175 ceph-mon[89733]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:03.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:33:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:33:03] "GET /metrics HTTP/1.1" 200 38483 "" "Prometheus/2.43.0" 2024-05-25T06:33:04.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:04 smithi179 ceph-mon[67580]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:04.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:04 smithi175 ceph-mon[84648]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:04.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:04 smithi175 ceph-mon[89733]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:05.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:05 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:05.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:05 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:06.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:05 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:06 smithi175 ceph-mon[84648]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:06.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:06 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:06 smithi175 ceph-mon[89733]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:06.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:06 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:06 smithi179 ceph-mon[67580]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:07.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:06 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:08.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:08 smithi175 ceph-mon[84648]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:08.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:08 smithi175 ceph-mon[89733]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:09.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:08 smithi179 ceph-mon[67580]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:10.819 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:10 smithi175 ceph-mon[89733]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:10.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:10 smithi175 ceph-mon[84648]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:11.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:10 smithi179 ceph-mon[67580]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:12.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:12 smithi175 ceph-mon[89733]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:12.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:12 smithi175 ceph-mon[84648]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:13.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:12 smithi179 ceph-mon[67580]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:13.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:33:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:33:13] "GET /metrics HTTP/1.1" 200 38482 "" "Prometheus/2.43.0" 2024-05-25T06:33:14.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:14 smithi175 ceph-mon[84648]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:14.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:14 smithi175 ceph-mon[89733]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:15.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:14 smithi179 ceph-mon[67580]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:15.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:15 smithi175 ceph-mon[84648]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:15.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:15 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:15 smithi175 ceph-mon[89733]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:15.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:15 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:15 smithi179 ceph-mon[67580]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:16.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:15 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:18.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:18 smithi179 ceph-mon[67580]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:18.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:18 smithi175 ceph-mon[89733]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:18.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:18 smithi175 ceph-mon[84648]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:20.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:20 smithi179 ceph-mon[67580]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:20.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:20 smithi175 ceph-mon[84648]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:20.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:20 smithi175 ceph-mon[89733]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:21.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:21.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:21.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:22.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:22 smithi179 ceph-mon[67580]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:22.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:22 smithi175 ceph-mon[84648]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:22.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:22 smithi175 ceph-mon[89733]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:33:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:33:23] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-05-25T06:33:24.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:24 smithi179 ceph-mon[67580]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:24.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:24 smithi175 ceph-mon[84648]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:24.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:24 smithi175 ceph-mon[89733]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:26.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:26 smithi175 ceph-mon[84648]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:26.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:26 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:26.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:26 smithi175 ceph-mon[89733]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:26.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:26 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:26 smithi179 ceph-mon[67580]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:26 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:28.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:28 smithi175 ceph-mon[89733]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:28.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:28 smithi175 ceph-mon[84648]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:28 smithi179 ceph-mon[67580]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:30.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:30 smithi175 ceph-mon[89733]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:30 smithi175 ceph-mon[84648]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:30.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:30 smithi179 ceph-mon[67580]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:32.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:32 smithi175 ceph-mon[89733]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:32.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:32 smithi175 ceph-mon[84648]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:32.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:32 smithi179 ceph-mon[67580]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:33:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:33:33] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-05-25T06:33:34.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:34 smithi175 ceph-mon[84648]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:34.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:34 smithi175 ceph-mon[89733]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:34.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:34 smithi179 ceph-mon[67580]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:36.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:36 smithi175 ceph-mon[84648]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:36.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:36 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:36.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:36 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:36.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:36 smithi175 ceph-mon[89733]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:36.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:36 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:36.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:36 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:36 smithi179 ceph-mon[67580]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:36 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:36.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:36 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:38.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:38 smithi175 ceph-mon[84648]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:38.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:38 smithi175 ceph-mon[89733]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:38.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:38 smithi179 ceph-mon[67580]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[89733]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:33:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:33:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:33:40.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:33:40.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[84648]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:40.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:33:40.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:33:40.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:33:40.571 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:40 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:33:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:40 smithi179 ceph-mon[67580]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:40 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-25T06:33:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:40 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-25T06:33:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:40 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-25T06:33:40.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:40 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' 2024-05-25T06:33:42.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:42 smithi175 ceph-mon[89733]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:42.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:42 smithi175 ceph-mon[84648]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:42.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:42 smithi179 ceph-mon[67580]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:43.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:33:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:33:43] "GET /metrics HTTP/1.1" 200 38483 "" "Prometheus/2.43.0" 2024-05-25T06:33:44.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:44 smithi175 ceph-mon[89733]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:44.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:44 smithi175 ceph-mon[84648]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:44.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:44 smithi179 ceph-mon[67580]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:33:46.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:46 smithi175 ceph-mon[84648]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:46.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:46 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:46.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:46 smithi175 ceph-mon[89733]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:46.571 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:46 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:46 smithi179 ceph-mon[67580]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:46.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:46 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:48.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:48 smithi175 ceph-mon[89733]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:48.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:48 smithi175 ceph-mon[84648]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:48.587 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:48 smithi179 ceph-mon[67580]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:50.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:50 smithi179 ceph-mon[67580]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:50.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:50 smithi175 ceph-mon[84648]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:50.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:50 smithi175 ceph-mon[89733]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:51.721 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:51 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:51.721 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:51 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:51.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:51 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:33:52.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:52 smithi179 ceph-mon[67580]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:52.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:52 smithi175 ceph-mon[89733]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:52.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:52 smithi175 ceph-mon[84648]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:53.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:33:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:33:53] "GET /metrics HTTP/1.1" 200 38480 "" "Prometheus/2.43.0" 2024-05-25T06:33:54.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:54 smithi179 ceph-mon[67580]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:54.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:54 smithi175 ceph-mon[84648]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:54.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:54 smithi175 ceph-mon[89733]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:56 smithi179 ceph-mon[67580]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:56.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:56 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:56 smithi175 ceph-mon[84648]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:56.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:56 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:56.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:56 smithi175 ceph-mon[89733]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:33:56.821 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:56 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:33:58.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:33:58 smithi179 ceph-mon[67580]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:58.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:33:58 smithi175 ceph-mon[84648]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:33:58.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:33:58 smithi175 ceph-mon[89733]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:34:00.231 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ps' 2024-05-25T06:34:00.543 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:00 smithi175 ceph-mon[84648]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:00.544 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:00 smithi175 ceph-mon[89733]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:00.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:00 smithi179 ceph-mon[67580]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:02.377 INFO:teuthology.orchestra.run.smithi175.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-25T06:34:02.377 INFO:teuthology.orchestra.run.smithi175.stdout:alertmanager.a smithi175 *:9093,9094 running (12m) 83s ago 34m - - 0.25.0 c8568f914cd2 12c8cd2eac32 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:grafana.a smithi179 *:3000 running (11m) 2m ago 33m 82.1M - 9.4.12 2bacad6d85d8 fb91afb62378 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:iscsi.foo.smithi175.buback smithi175 running (87s) 83s ago 33m - - 3.8 f9a9809cafd6 64013f30a147 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.x smithi179 *:8443,9283,8765 running (13m) 2m ago 38m 454M - 19.0.0-2843-gea0684c0 f9a9809cafd6 ab279f96819d 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:mgr.y smithi175 *:8443,9283,8765 running (23m) 83s ago 40m - - 19.0.0-2843-gea0684c0 f9a9809cafd6 a07344169089 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:mon.a smithi175 running (9m) 83s ago 40m - 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 57a27054d700 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:mon.b smithi179 running (10m) 2m ago 38m 51.2M 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 8cb1d1e3627b 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:mon.c smithi175 running (9m) 83s ago 39m - 2048M 19.0.0-2843-gea0684c0 f9a9809cafd6 067b802471a7 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.a smithi175 *:9100 running (13m) 83s ago 34m - - 1.5.0 0da6a335fe13 e713992d4382 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:node-exporter.b smithi179 *:9100 running (12m) 2m ago 34m 15.0M - 1.5.0 0da6a335fe13 a3e7fafe20df 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:osd.0 smithi175 running (7m) 83s ago 38m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 e55ca2bd159e 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:osd.1 smithi175 running (6m) 83s ago 38m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 3d0004653079 2024-05-25T06:34:02.378 INFO:teuthology.orchestra.run.smithi175.stdout:osd.2 smithi175 running (5m) 83s ago 37m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 29ab74e627e1 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:osd.3 smithi175 running (4m) 83s ago 37m - 2992M 19.0.0-2843-gea0684c0 f9a9809cafd6 6e3e799cd1d8 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:osd.4 smithi179 running (4m) 2m ago 36m 52.2M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 cac9395d4e63 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:osd.5 smithi179 running (4m) 2m ago 36m 48.1M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 5a1d82fca8df 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:osd.6 smithi179 running (3m) 2m ago 36m 46.5M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 8f5beea10bda 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:osd.7 smithi179 running (3m) 2m ago 35m 47.9M 3248M 19.0.0-2843-gea0684c0 f9a9809cafd6 771fbf01687f 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:prometheus.a smithi179 *:9095 running (12m) 2m ago 35m 68.8M - 2.43.0 a07b618ecd1d 5aff147d2abb 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi175.iwhmky smithi175 *:8000 running (2m) 83s ago 33m - - 19.0.0-2843-gea0684c0 f9a9809cafd6 43b35e3d1335 2024-05-25T06:34:02.379 INFO:teuthology.orchestra.run.smithi175.stdout:rgw.foo.smithi179.hopgfy smithi179 *:8000 running (2m) 2m ago 33m 74.1M - 19.0.0-2843-gea0684c0 f9a9809cafd6 a799d7d558b1 2024-05-25T06:34:02.572 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:02 smithi175 ceph-mon[89733]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:34:02.572 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:02 smithi175 ceph-mon[84648]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:34:02.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:02 smithi179 ceph-mon[67580]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-25T06:34:03.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions' 2024-05-25T06:34:03.639 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:03 smithi175 ceph-mon[84648]: from='client.44922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:03.640 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:03 smithi175 ceph-mon[89733]: from='client.44922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:03.640 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:34:03] "GET /metrics HTTP/1.1" 200 38480 "" "Prometheus/2.43.0" 2024-05-25T06:34:03.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:03 smithi179 ceph-mon[67580]: from='client.44922 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:04.477 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:04 smithi175 ceph-mon[84648]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:04.477 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:04 smithi175 ceph-mon[89733]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:04.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:04 smithi179 ceph-mon[67580]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:05.293 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:34:05.293 INFO:teuthology.orchestra.run.smithi175.stdout: "mon": { 2024-05-25T06:34:05.293 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 3 2024-05-25T06:34:05.293 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:34:05.293 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr": { 2024-05-25T06:34:05.293 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:34:05.293 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: "osd": { 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 8 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw": { 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 2 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: "overall": { 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)": 15 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-05-25T06:34:05.294 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:34:05.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:05 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/3167650808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:05.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:05 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/3167650808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:05.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:05 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/3167650808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:05.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch upgrade status' 2024-05-25T06:34:06.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:06 smithi175 ceph-mon[84648]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:06 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:34:06.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:06 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:06 smithi175 ceph-mon[89733]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:06 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:34:06.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:06 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:06 smithi179 ceph-mon[67580]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:06 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:34:06.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:06 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:07.902 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:34:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "in_progress": true, 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "services_complete": [ 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "iscsi", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "osd", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "mgr", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "mon", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "rgw" 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "progress": "16/21 daemons upgraded", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed.", 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "is_paused": true 2024-05-25T06:34:07.903 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:34:08.565 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:08 smithi175 ceph-mon[84648]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:08.565 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:08 smithi175 ceph-mon[89733]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:08.591 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph health detail' 2024-05-25T06:34:08.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:08 smithi179 ceph-mon[67580]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:09.569 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:09 smithi175 ceph-mon[89733]: from='client.44934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:09.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:09 smithi175 ceph-mon[84648]: from='client.44934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:09.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:09 smithi179 ceph-mon[67580]: from='client.44934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:10.608 INFO:teuthology.orchestra.run.smithi175.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. 2024-05-25T06:34:10.608 INFO:teuthology.orchestra.run.smithi175.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi175.buback on host smithi175 failed. 2024-05-25T06:34:10.608 INFO:teuthology.orchestra.run.smithi175.stdout: Upgrade daemon: iscsi.foo.smithi175.buback: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi175.buback ... 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:Creating ceph-iscsi config... 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/iscsi-gateway.cfg 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:Write file: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/iscsi.foo.smithi175.buback/tcmu-runner-entrypoint.sh 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d0bfa7ec4\x2d1a5b\x2d11ef\x2dbc9b\x2dc7b262605968.slice/ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:Non-zero exit code 1 from systemctl start ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:systemctl: stderr Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:systemctl: stderr See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout:Traceback (most recent call last): 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout: return _run_code(code, main_globals, None, 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout: exec(code, run_globals) 2024-05-25T06:34:10.609 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5618, in 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 5606, in main 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3096, in command_deploy_from 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3131, in _common_deploy 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 3151, in _deploy_daemon_container 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1087, in deploy_daemon 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/__main__.py", line 1223, in deploy_daemon_units 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout: File "/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/cephadm.c78258a9946c350e44c9d652c4770e8895f8ed367a8290dd8611f4d0daaeccc5/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout:RuntimeError: Failed command: systemctl start ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback: Job for ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service failed because the control process exited with error code. 2024-05-25T06:34:10.610 INFO:teuthology.orchestra.run.smithi175.stdout:See "systemctl status ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" and "journalctl -xeu ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@iscsi.foo.smithi175.buback.service" for details. 2024-05-25T06:34:10.622 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:10 smithi175 ceph-mon[89733]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:10.622 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:10 smithi175 ceph-mon[84648]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:10.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:10 smithi179 ceph-mon[67580]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:11.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-05-25T06:34:11.699 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:11 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/2099928965' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:34:11.699 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:11 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/2099928965' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:34:11.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:11 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/2099928965' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-25T06:34:12.677 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:12 smithi175 ceph-mon[89733]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:12.677 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:12 smithi175 ceph-mon[84648]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:12.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:12 smithi179 ceph-mon[67580]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:13.487 INFO:teuthology.orchestra.run.smithi175.stdout:true 2024-05-25T06:34:13.820 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:34:13] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-05-25T06:34:14.322 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-05-25T06:34:14.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:14 smithi175 ceph-mon[84648]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:14.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:14 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/4207559067' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:14 smithi175 ceph-mon[89733]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:14.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:14 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/4207559067' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:14 smithi179 ceph-mon[67580]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:14.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:14 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/4207559067' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:16.627 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph version 19.0.0-2843-gea0684c0 (ea0684c08cbcf83f4af9930de2267b3563ac5f78) squid (rc)" 2024-05-25T06:34:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:16 smithi179 ceph-mon[67580]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:16.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:16 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:16.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:16 smithi175 ceph-mon[84648]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:16.820 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:16 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:16 smithi175 ceph-mon[89733]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:16.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:16 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:17.298 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -e sha1=ea0684c08cbcf83f4af9930de2267b3563ac5f78 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-05-25T06:34:17.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:17 smithi175 ceph-mon[84648]: from='client.? 172.21.15.175:0/1099862657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:17.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:17 smithi175 ceph-mon[89733]: from='client.? 172.21.15.175:0/1099862657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:17.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:17 smithi179 ceph-mon[67580]: from='client.? 172.21.15.175:0/1099862657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-25T06:34:18.564 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:18 smithi175 ceph-mon[89733]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:18.565 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:18 smithi175 ceph-mon[84648]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:18.686 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:18 smithi179 ceph-mon[67580]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:19.254 INFO:teuthology.orchestra.run.smithi175.stdout:osd 8 2m ago - 2024-05-25T06:34:19.921 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-25T06:34:19.933 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi175.front.sepia.ceph.com 2024-05-25T06:34:19.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-05-25T06:34:20.558 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:20 smithi175 ceph-mon[89733]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:20.558 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:20 smithi175 ceph-mon[89733]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:20.559 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:20 smithi175 ceph-mon[84648]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:20.559 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:20 smithi175 ceph-mon[84648]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:20 smithi179 ceph-mon[67580]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:20.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:20 smithi179 ceph-mon[67580]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:21.473 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:21 smithi175 ceph-mon[89733]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:34:21.474 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:21 smithi175 ceph-mon[84648]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:34:21.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:21 smithi179 ceph-mon[67580]: from='mgr.25663 172.21.15.175:0/2195401467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-25T06:34:22.591 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-05-25T06:34:22.591 INFO:teuthology.orchestra.run.smithi175.stdout: "image": "quay.io/ceph/ceph", 2024-05-25T06:34:22.591 INFO:teuthology.orchestra.run.smithi175.stdout: "registry": "quay.io", 2024-05-25T06:34:22.591 INFO:teuthology.orchestra.run.smithi175.stdout: "bare_image": "ceph/ceph", 2024-05-25T06:34:22.591 INFO:teuthology.orchestra.run.smithi175.stdout: "versions": [] 2024-05-25T06:34:22.591 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-05-25T06:34:22.751 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:22 smithi175 ceph-mon[84648]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:22.751 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:22 smithi175 ceph-mon[89733]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:22.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:22 smithi179 ceph-mon[67580]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:23.235 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-05-25T06:34:23.554 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:23 smithi175 ceph-mon[89733]: from='client.44964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:23.554 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:23 smithi175 ceph-mon[84648]: from='client.44964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:23.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:23 smithi179 ceph-mon[67580]: from='client.44964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:23.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:34:23] "GET /metrics HTTP/1.1" 200 38482 "" "Prometheus/2.43.0" 2024-05-25T06:34:24.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:24 smithi175 ceph-mon[84648]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:24.570 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:24 smithi175 ceph-mon[89733]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:24.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:24 smithi179 ceph-mon[67580]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:25.807 INFO:teuthology.orchestra.run.smithi175.stdout: "16.2.0", 2024-05-25T06:34:26.458 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:26 smithi175 ceph-mon[84648]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:26.459 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:26 smithi175 ceph-mon[84648]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:26.459 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:26 smithi175 ceph-mon[84648]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:26.459 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:26 smithi175 ceph-mon[89733]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:26.459 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:26 smithi175 ceph-mon[89733]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:26.459 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:26 smithi175 ceph-mon[89733]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:26.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-05-25T06:34:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:26 smithi179 ceph-mon[67580]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:26 smithi179 ceph-mon[67580]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-25T06:34:26.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:26 smithi179 ceph-mon[67580]: from='client.54751 -' entity='client.iscsi.foo.smithi175.buback' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-25T06:34:28.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:28 smithi179 ceph-mon[67580]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:28.819 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:28 smithi175 ceph-mon[84648]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:28.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:28 smithi175 ceph-mon[89733]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-25T06:34:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "v16.2.2", 2024-05-25T06:34:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "v16.2.2-20210505", 2024-05-25T06:34:29.547 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:29 smithi175 ceph-mon[89733]: from='client.54916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:29.547 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:29 smithi175 ceph-mon[84648]: from='client.54916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:29.785 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:29 smithi179 ceph-mon[67580]: from='client.54916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-25T06:34:29.823 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-25T06:34:29.832 INFO:tasks.cephadm:Teardown begin 2024-05-25T06:34:29.832 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-25T06:34:29.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-25T06:34:29.895 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-25T06:34:29.895 DEBUG:teuthology.orchestra.run.smithi175:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-25T06:34:29.917 DEBUG:teuthology.orchestra.run.smithi179:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-25T06:34:29.953 INFO:tasks.cephadm:Stopping all daemons... 2024-05-25T06:34:29.953 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-05-25T06:34:29.954 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a 2024-05-25T06:34:30.069 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:29 smithi175 systemd[1]: Stopping Ceph mon.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:30.569 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a[84644]: 2024-05-25T06:34:30.277+0000 7f91bf7b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a[84644]: 2024-05-25T06:34:30.277+0000 7f91bf7b4700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:30 smithi175 podman[135820]: 2024-05-25 06:34:30.348304478 +0000 UTC m=+0.203824843 container died 57a27054d700ee77e14d2992c9cf2ad901bd143253efaad849743bda58d2c15a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c) 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:34:30] ENGINE Bus STOPPING 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:34:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:34:30] ENGINE Bus STOPPED 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:34:30] ENGINE Bus STARTING 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: CherryPy Checker: 2024-05-25T06:34:30.570 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: The Application mounted at '' has an empty config. 2024-05-25T06:34:30.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:34:30.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:34:30] ENGINE Serving on http://:::9283 2024-05-25T06:34:30.571 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:30 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: [25/May/2024:06:34:30] ENGINE Bus STARTED 2024-05-25T06:34:31.002 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:30 smithi175 podman[135820]: 2024-05-25 06:34:30.709814223 +0000 UTC m=+0.565334579 container cleanup 57a27054d700ee77e14d2992c9cf2ad901bd143253efaad849743bda58d2c15a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-25T06:34:31.002 INFO:journalctl@ceph.mon.a.smithi175.stdout:May 25 06:34:30 smithi175 bash[135820]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-a 2024-05-25T06:34:31.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.a.service' 2024-05-25T06:34:31.182 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:34:31.182 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-05-25T06:34:31.182 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-05-25T06:34:31.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.c 2024-05-25T06:34:31.539 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:31 smithi175 systemd[1]: Stopping Ceph mon.c for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:31 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c[89728]: 2024-05-25T06:34:31.538+0000 7f321bdfb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:34:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:31 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c[89728]: 2024-05-25T06:34:31.538+0000 7f321bdfb700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-05-25T06:34:31.820 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:31 smithi175 podman[135952]: 2024-05-25 06:34:31.590550517 +0000 UTC m=+0.177828608 container died 067b802471a7cd0e06cf656a37bb77b0b6140d4401b2e87362fe105a383970cb (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:34:32.240 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:31 smithi175 podman[135952]: 2024-05-25 06:34:31.955701714 +0000 UTC m=+0.542979806 container cleanup 067b802471a7cd0e06cf656a37bb77b0b6140d4401b2e87362fe105a383970cb (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0) 2024-05-25T06:34:32.240 INFO:journalctl@ceph.mon.c.smithi175.stdout:May 25 06:34:31 smithi175 bash[135952]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-c 2024-05-25T06:34:32.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.c.service' 2024-05-25T06:34:32.420 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:34:32.421 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-05-25T06:34:32.421 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-05-25T06:34:32.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.b 2024-05-25T06:34:32.768 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:32 smithi179 systemd[1]: Stopping Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b[67576]: 2024-05-25T06:34:32.767+0000 7f0c3c352700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:34:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b[67576]: 2024-05-25T06:34:32.767+0000 7f0c3c352700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-05-25T06:34:33.035 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:32 smithi179 podman[100467]: 2024-05-25 06:34:32.901520746 +0000 UTC m=+0.274969808 container died 8cb1d1e3627b934755c29dac51d56b6fe9212ea8d5123847d638ce87c71099b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, ceph=True, org.label-schema.build-date=20240521, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-05-25T06:34:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:33 smithi179 podman[100467]: 2024-05-25 06:34:33.335456728 +0000 UTC m=+0.708905848 container cleanup 8cb1d1e3627b934755c29dac51d56b6fe9212ea8d5123847d638ce87c71099b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c) 2024-05-25T06:34:33.535 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:33 smithi179 bash[100467]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b 2024-05-25T06:34:33.808 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.b.service' 2024-05-25T06:34:33.819 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: ::ffff:172.21.15.179 - - [25/May/2024:06:34:33] "GET /metrics HTTP/1.1" 200 38482 "" "Prometheus/2.43.0" 2024-05-25T06:34:33.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:33 smithi179 podman[100481]: 2024-05-25 06:34:33.563940127 +0000 UTC m=+0.664794461 container remove 8cb1d1e3627b934755c29dac51d56b6fe9212ea8d5123847d638ce87c71099b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mon-b, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-25T06:34:33.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:33 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.b.service: Deactivated successfully. 2024-05-25T06:34:33.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:33 smithi179 systemd[1]: Stopped Ceph mon.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:34:33.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:May 25 06:34:33 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mon.b.service: Consumed 7.441s CPU time. 2024-05-25T06:34:33.854 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:34:33.854 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-05-25T06:34:33.854 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-05-25T06:34:33.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y 2024-05-25T06:34:34.143 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:33 smithi175 systemd[1]: Stopping Ceph mgr.y for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:34.461 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:34:34.143+0000 7f7f16b21700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:34:34.461 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:34 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y[63284]: 2024-05-25T06:34:34.143+0000 7f7f16b21700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-05-25T06:34:34.461 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:34 smithi175 podman[136072]: 2024-05-25 06:34:34.184959187 +0000 UTC m=+0.149263537 container died a0734416908919eacc2c8841632da2b866067412f28dba3a6dbd76d5782193ff (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True) 2024-05-25T06:34:34.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:34 smithi175 podman[136072]: 2024-05-25 06:34:34.560504585 +0000 UTC m=+0.524808929 container cleanup a0734416908919eacc2c8841632da2b866067412f28dba3a6dbd76d5782193ff (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-25T06:34:34.811 INFO:journalctl@ceph.mgr.y.smithi175.stdout:May 25 06:34:34 smithi175 bash[136072]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-y 2024-05-25T06:34:34.941 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.y.service' 2024-05-25T06:34:34.976 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:34:34.976 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-05-25T06:34:34.976 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-05-25T06:34:34.976 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x 2024-05-25T06:34:35.285 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:35 smithi179 systemd[1]: Stopping Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:35.574 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:35 smithi179 podman[100588]: 2024-05-25 06:34:35.313720567 +0000 UTC m=+0.141899942 container died ab279f96819df27da2850ac855d4da176e252e9723ce0c784a19b5a4bbcdcf2f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-05-25T06:34:35.916 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:35 smithi179 podman[100588]: 2024-05-25 06:34:35.732444723 +0000 UTC m=+0.560624092 container cleanup ab279f96819df27da2850ac855d4da176e252e9723ce0c784a19b5a4bbcdcf2f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-25T06:34:35.916 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:35 smithi179 bash[100588]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x 2024-05-25T06:34:36.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service' 2024-05-25T06:34:36.185 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:35 smithi179 podman[100600]: 2024-05-25 06:34:35.917325827 +0000 UTC m=+0.603700137 container remove ab279f96819df27da2850ac855d4da176e252e9723ce0c784a19b5a4bbcdcf2f (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-05-25T06:34:36.185 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:35 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:34:36.185 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:36 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-25T06:34:36.185 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:36 smithi179 systemd[1]: Stopped Ceph mgr.x for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:34:36.185 INFO:journalctl@ceph.mgr.x.smithi179.stdout:May 25 06:34:36 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@mgr.x.service: Consumed 23.274s CPU time. 2024-05-25T06:34:36.198 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:34:36.198 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-05-25T06:34:36.199 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-25T06:34:36.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.0 2024-05-25T06:34:36.539 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:36 smithi175 systemd[1]: Stopping Ceph osd.0 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:36.819 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[98103]: 2024-05-25T06:34:36.539+0000 7f90728e8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:34:36.820 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[98103]: 2024-05-25T06:34:36.539+0000 7f90728e8700 -1 osd.0 161 *** Got signal Terminated *** 2024-05-25T06:34:36.820 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:36 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0[98103]: 2024-05-25T06:34:36.539+0000 7f90728e8700 -1 osd.0 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:34:42.320 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:41 smithi175 podman[136190]: 2024-05-25 06:34:41.948722185 +0000 UTC m=+5.552535359 container died e55ca2bd159e08c4d177ac6392ee450cabf6314501ddd917429d241496aba784 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5) 2024-05-25T06:34:42.640 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:42 smithi175 podman[136190]: 2024-05-25 06:34:42.339498171 +0000 UTC m=+5.943311345 container cleanup e55ca2bd159e08c4d177ac6392ee450cabf6314501ddd917429d241496aba784 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-05-25T06:34:42.640 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:42 smithi175 bash[136190]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0 2024-05-25T06:34:42.966 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:42 smithi175 podman[136206]: 2024-05-25 06:34:42.641023659 +0000 UTC m=+0.694319953 container remove e55ca2bd159e08c4d177ac6392ee450cabf6314501ddd917429d241496aba784 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0) 2024-05-25T06:34:43.269 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:42 smithi175 podman[136296]: 2024-05-25 06:34:42.96681076 +0000 UTC m=+0.111033976 container create 915f4b65f94bb5db204ad71a5c34a05216ec0c7e1e011326f914d7d2b6017e8a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-25T06:34:43.269 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:42 smithi175 podman[136296]: 2024-05-25 06:34:42.872562444 +0000 UTC m=+0.016785673 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:34:43.269 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:43 smithi175 podman[136296]: 2024-05-25 06:34:43.185850093 +0000 UTC m=+0.330073322 container init 915f4b65f94bb5db204ad71a5c34a05216ec0c7e1e011326f914d7d2b6017e8a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-25T06:34:43.269 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:43 smithi175 podman[136296]: 2024-05-25 06:34:43.190414406 +0000 UTC m=+0.334637623 container start 915f4b65f94bb5db204ad71a5c34a05216ec0c7e1e011326f914d7d2b6017e8a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-05-25T06:34:43.269 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:43 smithi175 podman[136296]: 2024-05-25 06:34:43.26912197 +0000 UTC m=+0.413345188 container attach 915f4b65f94bb5db204ad71a5c34a05216ec0c7e1e011326f914d7d2b6017e8a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-05-25T06:34:43.570 INFO:journalctl@ceph.osd.0.smithi175.stdout:May 25 06:34:43 smithi175 podman[136296]: 2024-05-25 06:34:43.479872098 +0000 UTC m=+0.624095315 container died 915f4b65f94bb5db204ad71a5c34a05216ec0c7e1e011326f914d7d2b6017e8a (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-25T06:34:44.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.0.service' 2024-05-25T06:34:44.099 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:34:44.100 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-25T06:34:44.100 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-25T06:34:44.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.1 2024-05-25T06:34:44.320 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:44 smithi175 systemd[1]: Stopping Ceph osd.1 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:44.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[103075]: 2024-05-25T06:34:44.485+0000 7f5675984700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:34:44.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[103075]: 2024-05-25T06:34:44.485+0000 7f5675984700 -1 osd.1 161 *** Got signal Terminated *** 2024-05-25T06:34:44.820 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1[103075]: 2024-05-25T06:34:44.485+0000 7f5675984700 -1 osd.1 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:34:50.299 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:50 smithi175 podman[136402]: 2024-05-25 06:34:50.00619515 +0000 UTC m=+5.627724973 container died 3d00046530798dd980198f4e03204ac5714087b21fcd3af56318ef99c0d079ea (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:34:50.570 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:50 smithi175 podman[136402]: 2024-05-25 06:34:50.300353969 +0000 UTC m=+5.921883791 container cleanup 3d00046530798dd980198f4e03204ac5714087b21fcd3af56318ef99c0d079ea (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-25T06:34:50.570 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:50 smithi175 bash[136402]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1 2024-05-25T06:34:50.570 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:50 smithi175 podman[136428]: 2024-05-25 06:34:50.526446008 +0000 UTC m=+0.511733546 container remove 3d00046530798dd980198f4e03204ac5714087b21fcd3af56318ef99c0d079ea (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, ceph=True, GIT_BRANCH=HEAD) 2024-05-25T06:34:51.100 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:50 smithi175 podman[136520]: 2024-05-25 06:34:50.751816228 +0000 UTC m=+0.017185300 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:34:51.100 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:50 smithi175 podman[136520]: 2024-05-25 06:34:50.86062995 +0000 UTC m=+0.125998996 container create 062f796c92da5789b2c6d42fdee919ee90eb3111636ed4601dae5c69d4c1bfde (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240521, GIT_CLEAN=True, RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-25T06:34:51.100 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:51 smithi175 podman[136520]: 2024-05-25 06:34:51.097048258 +0000 UTC m=+0.362417318 container init 062f796c92da5789b2c6d42fdee919ee90eb3111636ed4601dae5c69d4c1bfde (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:34:51.100 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:51 smithi175 podman[136520]: 2024-05-25 06:34:51.100651516 +0000 UTC m=+0.366020578 container start 062f796c92da5789b2c6d42fdee919ee90eb3111636ed4601dae5c69d4c1bfde (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, GIT_CLEAN=True, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:34:51.400 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:51 smithi175 podman[136520]: 2024-05-25 06:34:51.213427892 +0000 UTC m=+0.478796957 container attach 062f796c92da5789b2c6d42fdee919ee90eb3111636ed4601dae5c69d4c1bfde (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2) 2024-05-25T06:34:51.666 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:51 smithi175 podman[136520]: 2024-05-25 06:34:51.401671707 +0000 UTC m=+0.667040763 container died 062f796c92da5789b2c6d42fdee919ee90eb3111636ed4601dae5c69d4c1bfde (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2) 2024-05-25T06:34:51.922 INFO:journalctl@ceph.osd.1.smithi175.stdout:May 25 06:34:51 smithi175 podman[136546]: 2024-05-25 06:34:51.908375184 +0000 UTC m=+0.500253663 container remove 062f796c92da5789b2c6d42fdee919ee90eb3111636ed4601dae5c69d4c1bfde (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-1-deactivate, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS) 2024-05-25T06:34:51.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.1.service' 2024-05-25T06:34:51.967 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:34:51.968 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-25T06:34:51.968 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-25T06:34:51.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.2 2024-05-25T06:34:52.302 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:52 smithi175 systemd[1]: Stopping Ceph osd.2 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:34:52.569 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[109885]: 2024-05-25T06:34:52.302+0000 7f0c88c63700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:34:52.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[109885]: 2024-05-25T06:34:52.302+0000 7f0c88c63700 -1 osd.2 161 *** Got signal Terminated *** 2024-05-25T06:34:52.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:52 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2[109885]: 2024-05-25T06:34:52.302+0000 7f0c88c63700 -1 osd.2 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:34:53.819 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:34:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:34:53.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:34:53.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:34:53 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:34:53.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:34:58.070 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:57 smithi175 podman[136626]: 2024-05-25 06:34:57.814908144 +0000 UTC m=+5.618982601 container died 29ab74e627e13978a8565a79055bdbbc6834c4653791ff485a0ad04bfb134f25 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0) 2024-05-25T06:34:58.453 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:58 smithi175 podman[136626]: 2024-05-25 06:34:58.260714727 +0000 UTC m=+6.064789226 container cleanup 29ab74e627e13978a8565a79055bdbbc6834c4653791ff485a0ad04bfb134f25 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c) 2024-05-25T06:34:58.453 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:58 smithi175 bash[136626]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2 2024-05-25T06:34:58.819 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:58 smithi175 podman[136643]: 2024-05-25 06:34:58.453903792 +0000 UTC m=+0.632596433 container remove 29ab74e627e13978a8565a79055bdbbc6834c4653791ff485a0ad04bfb134f25 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.build-date=20240521) 2024-05-25T06:34:59.199 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:58 smithi175 podman[136734]: 2024-05-25 06:34:58.7857213 +0000 UTC m=+0.017459258 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:34:59.199 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:58 smithi175 podman[136734]: 2024-05-25 06:34:58.904682467 +0000 UTC m=+0.136420406 container create 83c4d2c01187072c737172325622299e74817ae0dd56c2f41e1eb9f03a4245d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2) 2024-05-25T06:34:59.199 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:59 smithi175 podman[136734]: 2024-05-25 06:34:59.124026552 +0000 UTC m=+0.355764490 container init 83c4d2c01187072c737172325622299e74817ae0dd56c2f41e1eb9f03a4245d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:34:59.199 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:59 smithi175 podman[136734]: 2024-05-25 06:34:59.127272063 +0000 UTC m=+0.359010023 container start 83c4d2c01187072c737172325622299e74817ae0dd56c2f41e1eb9f03a4245d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:34:59.199 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:59 smithi175 podman[136734]: 2024-05-25 06:34:59.198872886 +0000 UTC m=+0.430610855 container attach 83c4d2c01187072c737172325622299e74817ae0dd56c2f41e1eb9f03a4245d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:34:59.570 INFO:journalctl@ceph.osd.2.smithi175.stdout:May 25 06:34:59 smithi175 podman[136734]: 2024-05-25 06:34:59.415245299 +0000 UTC m=+0.646983245 container died 83c4d2c01187072c737172325622299e74817ae0dd56c2f41e1eb9f03a4245d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240521) 2024-05-25T06:35:00.016 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.2.service' 2024-05-25T06:35:00.054 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:35:00.054 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-25T06:35:00.054 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-25T06:35:00.054 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.3 2024-05-25T06:35:00.319 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:00 smithi175 systemd[1]: Stopping Ceph osd.3 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:00.819 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:35:00.379+0000 7f78ff67c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:35:00.819 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:35:00.379+0000 7f78ff67c700 -1 osd.3 161 *** Got signal Terminated *** 2024-05-25T06:35:00.820 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:00 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:35:00.379+0000 7f78ff67c700 -1 osd.3 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:35:03.705 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:03 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:03.428+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:03.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:03.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:35:03.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:03 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:03.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:04.319 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:35:04.029+0000 7f78f7c68700 -1 osd.3 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.887036+0000 front 2024-05-25T06:34:37.887081+0000 (oldest deadline 2024-05-25T06:35:03.187177+0000) 2024-05-25T06:35:04.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:04.209+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:04.535 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:04.451+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:05.215 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:04 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:04.985+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:05.319 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:04 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3[117087]: 2024-05-25T06:35:04.991+0000 7f78f7c68700 -1 osd.3 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.887036+0000 front 2024-05-25T06:34:37.887081+0000 (oldest deadline 2024-05-25T06:35:03.187177+0000) 2024-05-25T06:35:05.469 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:05.216+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:05.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:05 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:05.469+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:06.162 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:05 smithi175 podman[136839]: 2024-05-25 06:35:05.871096405 +0000 UTC m=+5.599770447 container died 6e3e799cd1d8c3f219512bd1d88d2a52418d2df50d02f8050b5e389f08d8f2aa (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-05-25T06:35:06.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:06 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:06.217+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:06.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:06 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:06.031+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:06.504 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:06 smithi175 podman[136839]: 2024-05-25 06:35:06.254195657 +0000 UTC m=+5.982869700 container cleanup 6e3e799cd1d8c3f219512bd1d88d2a52418d2df50d02f8050b5e389f08d8f2aa (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True) 2024-05-25T06:35:06.505 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:06 smithi175 bash[136839]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3 2024-05-25T06:35:06.779 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:06 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:06.484+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:06.819 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:06 smithi175 podman[136857]: 2024-05-25 06:35:06.505286523 +0000 UTC m=+0.626015291 container remove 6e3e799cd1d8c3f219512bd1d88d2a52418d2df50d02f8050b5e389f08d8f2aa (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-25T06:35:07.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:06 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:06.780+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:07.132 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:06 smithi175 podman[136944]: 2024-05-25 06:35:06.734031186 +0000 UTC m=+0.018062367 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:35:07.132 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:06 smithi175 podman[136944]: 2024-05-25 06:35:06.831042372 +0000 UTC m=+0.115073546 container create bf6f01b37a13d34d96001ce7928304831f248f9060597ba1d0bba9f3feafa6d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-25T06:35:07.132 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:07 smithi175 podman[136944]: 2024-05-25 06:35:07.049890865 +0000 UTC m=+0.333922036 container init bf6f01b37a13d34d96001ce7928304831f248f9060597ba1d0bba9f3feafa6d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-05-25T06:35:07.132 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:07 smithi175 podman[136944]: 2024-05-25 06:35:07.053070388 +0000 UTC m=+0.337101566 container start bf6f01b37a13d34d96001ce7928304831f248f9060597ba1d0bba9f3feafa6d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True) 2024-05-25T06:35:07.133 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:07 smithi175 podman[136944]: 2024-05-25 06:35:07.133174077 +0000 UTC m=+0.417205251 container attach bf6f01b37a13d34d96001ce7928304831f248f9060597ba1d0bba9f3feafa6d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-25T06:35:07.514 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:07.248+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:07.515 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:07.061+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:07.569 INFO:journalctl@ceph.osd.3.smithi175.stdout:May 25 06:35:07 smithi175 podman[136944]: 2024-05-25 06:35:07.346667742 +0000 UTC m=+0.630698918 container died bf6f01b37a13d34d96001ce7928304831f248f9060597ba1d0bba9f3feafa6d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-25T06:35:07.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:07.514+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:07.785 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:07 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:07.732+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:07.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.3.service' 2024-05-25T06:35:07.980 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:35:07.981 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-25T06:35:07.981 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-25T06:35:07.981 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.4 2024-05-25T06:35:08.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:08.030+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:08.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:08 smithi179 systemd[1]: Stopping Ceph osd.4 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:08.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:08.240+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:08.732 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:08.401+0000 7fa2b95ed700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:35:08.732 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:08.401+0000 7fa2b95ed700 -1 osd.4 161 *** Got signal Terminated *** 2024-05-25T06:35:08.732 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:08.401+0000 7fa2b95ed700 -1 osd.4 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:35:08.732 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:08.548+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:09.023 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:08 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:08.732+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:09.278 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:09.023+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:09.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:09.278+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:09.981 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:09.778+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:09.981 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:09.552+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:10.248 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:09 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:09.981+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:10.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:10 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:10.249+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:10.944 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:10 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:10.815+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:10.944 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:10 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:10.584+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:11.230 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:10 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:10.944+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:11.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:11.231+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:11.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:11.231+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:48.782080+0000 front 2024-05-25T06:34:48.782049+0000 (oldest deadline 2024-05-25T06:35:11.081877+0000) 2024-05-25T06:35:11.975 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:11.807+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:11.976 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:11.592+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:12.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:11.976+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:12.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:12 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:12.203+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:12.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:12 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:12.203+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:48.782080+0000 front 2024-05-25T06:34:48.782049+0000 (oldest deadline 2024-05-25T06:35:11.081877+0000) 2024-05-25T06:35:13.014 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:12 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:12.635+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:13.014 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:12 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:12.792+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:13.014 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:12 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:12.792+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:13.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:13.185+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:38.880807+0000 front 2024-05-25T06:34:38.880707+0000 (oldest deadline 2024-05-25T06:35:03.580590+0000) 2024-05-25T06:35:13.285 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4[77768]: 2024-05-25T06:35:13.185+0000 7fa2b5be1700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:48.782080+0000 front 2024-05-25T06:34:48.782049+0000 (oldest deadline 2024-05-25T06:35:11.081877+0000) 2024-05-25T06:35:13.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:13.014+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:13.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:13.014+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:13.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:13.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:35:13.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:13 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:13.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:14.035 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:13 smithi179 podman[100722]: 2024-05-25 06:35:13.73033326 +0000 UTC m=+5.549091671 container died cac9395d4e632609b549e45d73373db5400c30e3782092c1b473daf7088e7c41 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-25T06:35:14.036 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:13.620+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:14.036 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:13.620+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:14.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:13.793+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:14.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:13 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:13.793+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:14.361 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:14.053+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:14.361 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:14.053+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:14.361 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:14 smithi179 podman[100722]: 2024-05-25 06:35:14.218612533 +0000 UTC m=+6.037370926 container cleanup cac9395d4e632609b549e45d73373db5400c30e3782092c1b473daf7088e7c41 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-25T06:35:14.362 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:14 smithi179 bash[100722]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4 2024-05-25T06:35:14.785 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:14 smithi179 podman[100737]: 2024-05-25 06:35:14.428653664 +0000 UTC m=+0.688418653 container remove cac9395d4e632609b549e45d73373db5400c30e3782092c1b473daf7088e7c41 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-25T06:35:14.786 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:14.600+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:14.786 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:14.600+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:15.156 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:15.071+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:15.156 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:15.071+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:15.156 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:14.826+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:15.157 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:14 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:14.826+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:15.157 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:14 smithi179 podman[100830]: 2024-05-25 06:35:14.743910612 +0000 UTC m=+0.017463208 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:35:15.157 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:14 smithi179 podman[100830]: 2024-05-25 06:35:14.863039716 +0000 UTC m=+0.136592354 container create ed2263f2345974da0ff153d48208e10c38718fd71674975f30cad588798791fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2) 2024-05-25T06:35:15.157 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:15 smithi179 podman[100830]: 2024-05-25 06:35:15.07371452 +0000 UTC m=+0.347267109 container init ed2263f2345974da0ff153d48208e10c38718fd71674975f30cad588798791fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, ceph=True, GIT_CLEAN=True, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0) 2024-05-25T06:35:15.157 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:15 smithi179 podman[100830]: 2024-05-25 06:35:15.076988416 +0000 UTC m=+0.350541014 container start ed2263f2345974da0ff153d48208e10c38718fd71674975f30cad588798791fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-05-25T06:35:15.157 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:15 smithi179 podman[100830]: 2024-05-25 06:35:15.157161434 +0000 UTC m=+0.430714033 container attach ed2263f2345974da0ff153d48208e10c38718fd71674975f30cad588798791fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-05-25T06:35:15.535 INFO:journalctl@ceph.osd.4.smithi179.stdout:May 25 06:35:15 smithi179 podman[100830]: 2024-05-25 06:35:15.364481837 +0000 UTC m=+0.638034434 container died ed2263f2345974da0ff153d48208e10c38718fd71674975f30cad588798791fe (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.vendor=CentOS) 2024-05-25T06:35:15.977 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:15.623+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:15.978 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:15.623+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:15.978 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:15.846+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:15.978 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:15 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:15.846+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:16.001 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.4.service' 2024-05-25T06:35:16.039 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:35:16.039 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-25T06:35:16.039 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-25T06:35:16.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.5 2024-05-25T06:35:16.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:16.085+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:16.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:16.085+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:16.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:16 smithi179 systemd[1]: Stopping Ceph osd.5 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:16.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:16.587+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:16.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:16.587+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:16.785 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:16.389+0000 7faf95148700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:35:16.785 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:16.389+0000 7faf95148700 -1 osd.5 161 *** Got signal Terminated *** 2024-05-25T06:35:16.785 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:16.389+0000 7faf95148700 -1 osd.5 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:35:17.109 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:16.847+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:17.109 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:16 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:16.847+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:17.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:17.108+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:17.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:17.108+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:17.869 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:17.556+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:17.869 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:17.556+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:18.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:17.868+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:18.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:17 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:17.868+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:18.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:18.110+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:18.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:18.110+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:18.912 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:18.561+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:18.912 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:18.561+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:18.912 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:18.561+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:19.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:19.127+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:19.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:19.127+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:19.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:18.911+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:19.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:18.911+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:19.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:18 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:18.911+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:19.863 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:19.583+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:19.863 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:19.583+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:19.863 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:19.583+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:20.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:20.083+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:20.285 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:20.083+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:20.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:19.863+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:20.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:19.863+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:20.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:19 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:19.863+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:21.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:20.624+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:21.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:20.624+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:21.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:20.624+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:21.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:20.871+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:21.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:20.871+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:21.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:20 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:20.871+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:21.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:21.070+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:40.968254+0000 front 2024-05-25T06:34:40.968113+0000 (oldest deadline 2024-05-25T06:35:04.467922+0000) 2024-05-25T06:35:21.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:21.070+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:44.468451+0000 front 2024-05-25T06:34:44.468380+0000 (oldest deadline 2024-05-25T06:35:12.068725+0000) 2024-05-25T06:35:21.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5[82229]: 2024-05-25T06:35:21.070+0000 7faf9173c700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:54.369798+0000 front 2024-05-25T06:34:54.369769+0000 (oldest deadline 2024-05-25T06:35:20.269654+0000) 2024-05-25T06:35:21.906 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:21.644+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:21.906 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:21.644+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:21.907 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:21.644+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:22.206 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:21 smithi179 podman[100947]: 2024-05-25 06:35:21.921673903 +0000 UTC m=+5.658791556 container died 5a1d82fca8df5799cc88ee63b0d7e01214343bc711da617f4ee0cd4ea52f3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-05-25T06:35:22.207 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:21.906+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:22.207 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:21.906+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:22.207 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:21 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:21.906+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:22.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:22 smithi179 podman[100947]: 2024-05-25 06:35:22.414933252 +0000 UTC m=+6.152050899 container cleanup 5a1d82fca8df5799cc88ee63b0d7e01214343bc711da617f4ee0cd4ea52f3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-ea0684c, ceph=True, org.label-schema.build-date=20240521, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:35:22.535 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:22 smithi179 bash[100947]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5 2024-05-25T06:35:22.925 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:22 smithi179 podman[100963]: 2024-05-25 06:35:22.64177524 +0000 UTC m=+0.721386966 container remove 5a1d82fca8df5799cc88ee63b0d7e01214343bc711da617f4ee0cd4ea52f3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5, RELEASE=squid-ea0684c, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:35:22.925 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:22.647+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:22.925 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:22.647+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:22.925 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:22.647+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:23.194 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:22.925+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:23.194 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:22.925+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:23.194 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:22 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:22.925+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:23.195 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:22 smithi179 podman[101055]: 2024-05-25 06:35:22.874144957 +0000 UTC m=+0.017673779 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:35:23.195 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:22 smithi179 podman[101055]: 2024-05-25 06:35:22.984540976 +0000 UTC m=+0.128069831 container create 30a3b019eda216e156b8429a0fcef02617e0a6d3a162ccc32b8b161f044ef739 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240521, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, io.buildah.version=1.33.5) 2024-05-25T06:35:23.195 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:23 smithi179 podman[101055]: 2024-05-25 06:35:23.195185917 +0000 UTC m=+0.338714737 container init 30a3b019eda216e156b8429a0fcef02617e0a6d3a162ccc32b8b161f044ef739 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-25T06:35:23.491 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:23 smithi179 podman[101055]: 2024-05-25 06:35:23.198440799 +0000 UTC m=+0.341969609 container start 30a3b019eda216e156b8429a0fcef02617e0a6d3a162ccc32b8b161f044ef739 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:35:23.491 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:23 smithi179 podman[101055]: 2024-05-25 06:35:23.295190596 +0000 UTC m=+0.438719419 container attach 30a3b019eda216e156b8429a0fcef02617e0a6d3a162ccc32b8b161f044ef739 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-05-25T06:35:23.781 INFO:journalctl@ceph.osd.5.smithi179.stdout:May 25 06:35:23 smithi179 podman[101055]: 2024-05-25 06:35:23.492282007 +0000 UTC m=+0.635810831 container died 30a3b019eda216e156b8429a0fcef02617e0a6d3a162ccc32b8b161f044ef739 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240521, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-05-25T06:35:23.781 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:23.672+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:23.781 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:23.672+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:23.782 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:23.672+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:23.819 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:23.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:23.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:35:23.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:23 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:23.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:24.031 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:23.899+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:24.031 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:23.899+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:24.032 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:23 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:23.899+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:24.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.5.service' 2024-05-25T06:35:24.090 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:35:24.090 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-25T06:35:24.091 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-25T06:35:24.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.6 2024-05-25T06:35:24.285 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:24 smithi179 systemd[1]: Stopping Ceph osd.6 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:24.684 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:24.426+0000 7ff66bfe5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:35:24.684 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:24.426+0000 7ff66bfe5700 -1 osd.6 161 *** Got signal Terminated *** 2024-05-25T06:35:24.684 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:24.426+0000 7ff66bfe5700 -1 osd.6 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:35:25.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:24.684+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:25.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:24.684+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:25.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:24.684+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:25.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:24.932+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:25.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:24.932+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:25.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:24 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:24.932+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:26.024 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:25.660+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:26.024 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:25.660+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:26.024 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:25.660+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:26.024 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:25.890+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:26.024 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:25.890+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:26.025 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:25 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:25.890+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:27.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:26.643+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:27.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:26.643+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:27.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:26.643+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:27.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:26.850+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:27.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:26.850+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:27.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:26 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:26.850+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:28.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:27.638+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:28.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:27.638+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:28.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:27.638+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:28.036 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:27.638+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:01.508786+0000 front 2024-05-25T06:35:01.508768+0000 (oldest deadline 2024-05-25T06:35:26.808711+0000) 2024-05-25T06:35:28.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:27.818+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:28.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:27.818+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:28.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:27 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:27.818+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:29.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:28.682+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:41.404595+0000 front 2024-05-25T06:34:41.404423+0000 (oldest deadline 2024-05-25T06:35:02.504545+0000) 2024-05-25T06:35:29.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:28.682+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:49.406434+0000 front 2024-05-25T06:34:49.406494+0000 (oldest deadline 2024-05-25T06:35:13.506208+0000) 2024-05-25T06:35:29.035 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:28.682+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:55.807460+0000 front 2024-05-25T06:34:55.807435+0000 (oldest deadline 2024-05-25T06:35:18.107448+0000) 2024-05-25T06:35:29.036 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6[86874]: 2024-05-25T06:35:28.682+0000 7ff6685d9700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:01.508786+0000 front 2024-05-25T06:35:01.508768+0000 (oldest deadline 2024-05-25T06:35:26.808711+0000) 2024-05-25T06:35:29.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:28.819+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:29.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:28.819+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:29.036 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:28 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:28.819+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:30.227 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:29 smithi179 podman[101161]: 2024-05-25 06:35:29.959597375 +0000 UTC m=+5.650186511 container died 8f5beea10bda4d378c3e93778ffa2f47125a501dfc5045e51e75e1a0d32e22d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-25T06:35:30.228 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:29.852+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:30.228 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:29.852+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:30.228 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:29 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:29.852+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:30.510 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:30 smithi179 podman[101161]: 2024-05-25 06:35:30.310466095 +0000 UTC m=+6.001055203 container cleanup 8f5beea10bda4d378c3e93778ffa2f47125a501dfc5045e51e75e1a0d32e22d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-05-25T06:35:30.511 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:30 smithi179 bash[101161]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6 2024-05-25T06:35:30.785 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:30 smithi179 podman[101178]: 2024-05-25 06:35:30.511641376 +0000 UTC m=+0.546197259 container remove 8f5beea10bda4d378c3e93778ffa2f47125a501dfc5045e51e75e1a0d32e22d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-05-25T06:35:31.139 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:30.816+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:31.139 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:30.816+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:31.139 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:30.817+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:31.140 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:30 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:30.817+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:05.075419+0000 front 2024-05-25T06:35:05.075359+0000 (oldest deadline 2024-05-25T06:35:30.375279+0000) 2024-05-25T06:35:31.140 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:30 smithi179 podman[101268]: 2024-05-25 06:35:30.829283954 +0000 UTC m=+0.102871341 container create 7a0b134a445f6e29cb6fa63ce7922c5665af431c49502c6622f469b6680dfc48 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-05-25T06:35:31.140 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:30 smithi179 podman[101268]: 2024-05-25 06:35:30.743771555 +0000 UTC m=+0.017358938 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:35:31.140 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:31 smithi179 podman[101268]: 2024-05-25 06:35:31.057144069 +0000 UTC m=+0.330731439 container init 7a0b134a445f6e29cb6fa63ce7922c5665af431c49502c6622f469b6680dfc48 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-25T06:35:31.140 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:31 smithi179 podman[101268]: 2024-05-25 06:35:31.060631573 +0000 UTC m=+0.334218943 container start 7a0b134a445f6e29cb6fa63ce7922c5665af431c49502c6622f469b6680dfc48 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, org.label-schema.build-date=20240521, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-25T06:35:31.140 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:31 smithi179 podman[101268]: 2024-05-25 06:35:31.140205295 +0000 UTC m=+0.413792692 container attach 7a0b134a445f6e29cb6fa63ce7922c5665af431c49502c6622f469b6680dfc48 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-ea0684c, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-25T06:35:31.535 INFO:journalctl@ceph.osd.6.smithi179.stdout:May 25 06:35:31 smithi179 podman[101268]: 2024-05-25 06:35:31.35342895 +0000 UTC m=+0.627016318 container died 7a0b134a445f6e29cb6fa63ce7922c5665af431c49502c6622f469b6680dfc48 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-6-deactivate, RELEASE=squid-ea0684c, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True) 2024-05-25T06:35:31.917 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:31.825+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:31.918 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:31.825+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:31.918 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:31.825+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:31.918 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:31 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:31.825+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:05.075419+0000 front 2024-05-25T06:35:05.075359+0000 (oldest deadline 2024-05-25T06:35:30.375279+0000) 2024-05-25T06:35:31.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.6.service' 2024-05-25T06:35:31.982 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:35:31.982 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-25T06:35:31.983 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-25T06:35:31.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.7 2024-05-25T06:35:32.220 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 systemd[1]: Stopping Ceph osd.7 for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:32.535 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:32.313+0000 7f487c17e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-25T06:35:32.535 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:32.313+0000 7f487c17e700 -1 osd.7 161 *** Got signal Terminated *** 2024-05-25T06:35:32.535 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:32.313+0000 7f487c17e700 -1 osd.7 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-25T06:35:32.535 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:32.220Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:35:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:32.225Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:35:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:32.225Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:35:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:32.225Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:35:32.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:32.225Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.175:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.175:8765: connect: connection refused" 2024-05-25T06:35:33.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:32.870+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:33.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:32.870+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:33.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:32.870+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:33.286 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:32 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:32.870+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:05.075419+0000 front 2024-05-25T06:35:05.075359+0000 (oldest deadline 2024-05-25T06:35:30.375279+0000) 2024-05-25T06:35:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:33.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:35:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:33.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:35:33.820 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:33 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:33.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:34.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:33.882+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:34.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:33.882+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:34.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:33.882+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:34.035 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:33 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:33.882+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:05.075419+0000 front 2024-05-25T06:35:05.075359+0000 (oldest deadline 2024-05-25T06:35:30.375279+0000) 2024-05-25T06:35:35.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:34 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:34.893+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:35.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:34 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:34.893+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:35.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:34 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:34.893+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:35.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:34 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:34.893+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:05.075419+0000 front 2024-05-25T06:35:05.075359+0000 (oldest deadline 2024-05-25T06:35:30.375279+0000) 2024-05-25T06:35:36.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:35.849+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:36.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:35.849+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:36.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:35.849+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:36.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:35.849+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:05.075419+0000 front 2024-05-25T06:35:05.075359+0000 (oldest deadline 2024-05-25T06:35:30.375279+0000) 2024-05-25T06:35:36.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:35 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:35.849+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.179:6804 osd.4 since back 2024-05-25T06:35:10.375762+0000 front 2024-05-25T06:35:10.375688+0000 (oldest deadline 2024-05-25T06:35:35.675619+0000) 2024-05-25T06:35:37.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:36 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:36.869+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6806 osd.0 since back 2024-05-25T06:34:37.671886+0000 front 2024-05-25T06:34:37.672068+0000 (oldest deadline 2024-05-25T06:35:06.472540+0000) 2024-05-25T06:35:37.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:36 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:36.869+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6814 osd.1 since back 2024-05-25T06:34:46.473000+0000 front 2024-05-25T06:34:46.473130+0000 (oldest deadline 2024-05-25T06:35:12.372857+0000) 2024-05-25T06:35:37.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:36 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:36.869+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6822 osd.2 since back 2024-05-25T06:34:53.473841+0000 front 2024-05-25T06:34:53.473857+0000 (oldest deadline 2024-05-25T06:35:18.173723+0000) 2024-05-25T06:35:37.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:36 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:36.869+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.175:6830 osd.3 since back 2024-05-25T06:35:05.075419+0000 front 2024-05-25T06:35:05.075359+0000 (oldest deadline 2024-05-25T06:35:30.375279+0000) 2024-05-25T06:35:37.285 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:36 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7[91358]: 2024-05-25T06:35:36.869+0000 7f4878772700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.179:6804 osd.4 since back 2024-05-25T06:35:10.375762+0000 front 2024-05-25T06:35:10.375688+0000 (oldest deadline 2024-05-25T06:35:35.675619+0000) 2024-05-25T06:35:38.047 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:37 smithi179 podman[101374]: 2024-05-25 06:35:37.797321562 +0000 UTC m=+5.588316209 container died 771fbf01687f07bc186852a6fb374f800341e4b6333a29b25b16e47de74b33a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240521, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:35:38.455 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:38 smithi179 podman[101374]: 2024-05-25 06:35:38.212918334 +0000 UTC m=+6.003912970 container cleanup 771fbf01687f07bc186852a6fb374f800341e4b6333a29b25b16e47de74b33a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240521, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-ea0684c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2) 2024-05-25T06:35:38.455 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:38 smithi179 bash[101374]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7 2024-05-25T06:35:38.785 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:38 smithi179 podman[101403]: 2024-05-25 06:35:38.456208912 +0000 UTC m=+0.649678512 container remove 771fbf01687f07bc186852a6fb374f800341e4b6333a29b25b16e47de74b33a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7, RELEASE=squid-ea0684c, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240521) 2024-05-25T06:35:39.192 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:38 smithi179 podman[101488]: 2024-05-25 06:35:38.890418207 +0000 UTC m=+0.108314428 container create 2b635a2365961ee591712cb5203a0ea7c6f9959b4cd585a663d107d80bd0a6d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521) 2024-05-25T06:35:39.192 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:38 smithi179 podman[101488]: 2024-05-25 06:35:38.799179376 +0000 UTC m=+0.017075619 image pull f9a9809cafd60a51ac0d352bb3d2d070ab25f705b0418cbd27290854c20186a2 quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee 2024-05-25T06:35:39.192 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:39 smithi179 podman[101488]: 2024-05-25 06:35:39.117772114 +0000 UTC m=+0.335668343 container init 2b635a2365961ee591712cb5203a0ea7c6f9959b4cd585a663d107d80bd0a6d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-25T06:35:39.192 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:39 smithi179 podman[101488]: 2024-05-25 06:35:39.121018168 +0000 UTC m=+0.338914409 container start 2b635a2365961ee591712cb5203a0ea7c6f9959b4cd585a663d107d80bd0a6d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240521, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-ea0684c, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-25T06:35:39.193 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:39 smithi179 podman[101488]: 2024-05-25 06:35:39.192732094 +0000 UTC m=+0.410628337 container attach 2b635a2365961ee591712cb5203a0ea7c6f9959b4cd585a663d107d80bd0a6d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240521, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093) 2024-05-25T06:35:39.535 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:39 smithi179 podman[101488]: 2024-05-25 06:35:39.417588078 +0000 UTC m=+0.635484352 container died 2b635a2365961ee591712cb5203a0ea7c6f9959b4cd585a663d107d80bd0a6d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, RELEASE=squid-ea0684c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240521, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-25T06:35:39.997 INFO:journalctl@ceph.osd.7.smithi179.stdout:May 25 06:35:39 smithi179 podman[101515]: 2024-05-25 06:35:39.996429018 +0000 UTC m=+0.572565999 container remove 2b635a2365961ee591712cb5203a0ea7c6f9959b4cd585a663d107d80bd0a6d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:87d6178d73fd8ae7a5c0bfa0c0b94e94dac0cc7c0eff96d5e0b3e0bddc5affee, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-ea0684c, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240521, GIT_COMMIT=e5230d3b004eecd2f42a9b578ebb9642654e8093, org.label-schema.license=GPLv2) 2024-05-25T06:35:40.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@osd.7.service' 2024-05-25T06:35:40.052 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:35:40.053 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-25T06:35:40.053 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-05-25T06:35:40.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a 2024-05-25T06:35:40.285 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 systemd[1]: Stopping Ceph prometheus.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:40.641 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.374Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:35:40.641 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.375Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.376Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.376Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a[63784]: ts=2024-05-25T06:35:40.376Z caller=main.go:1147 level=info msg="See you next time!" 2024-05-25T06:35:40.642 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 podman[101595]: 2024-05-25 06:35:40.385421599 +0000 UTC m=+0.108654685 container died 5aff147d2abb5776e4cfb532300a5a357bf3b72976b763cc9ceb197b798be36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:41.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 podman[101595]: 2024-05-25 06:35:40.833313352 +0000 UTC m=+0.556546427 container cleanup 5aff147d2abb5776e4cfb532300a5a357bf3b72976b763cc9ceb197b798be36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:41.035 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:May 25 06:35:40 smithi179 bash[101595]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-prometheus-a 2024-05-25T06:35:41.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@prometheus.a.service' 2024-05-25T06:35:41.284 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-25T06:35:41.284 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-05-25T06:35:41.285 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 --force --keep-logs 2024-05-25T06:35:43.045 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:42 smithi175 systemd[1]: Stopping Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:43.319 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 podman[137338]: 2024-05-25 06:35:43.04586297 +0000 UTC m=+0.125921625 container died e713992d438246b355ad47fb662cd36d5e78dbb72ed8effa57d25cabf29f5d42 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:43.752 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 podman[137338]: 2024-05-25 06:35:43.526276318 +0000 UTC m=+0.606334966 container cleanup e713992d438246b355ad47fb662cd36d5e78dbb72ed8effa57d25cabf29f5d42 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:43.752 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 bash[137338]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a 2024-05-25T06:35:43.752 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:43.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:43.752 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:43.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi175.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.175:8443: connect: connection refused" 2024-05-25T06:35:43.752 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:43 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:43.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi179.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.179:8443: connect: connection refused" 2024-05-25T06:35:44.069 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 podman[137350]: 2024-05-25 06:35:43.752412855 +0000 UTC m=+0.710273061 container remove e713992d438246b355ad47fb662cd36d5e78dbb72ed8effa57d25cabf29f5d42 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:44.069 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:35:44.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-25T06:35:44.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 systemd[1]: Stopped Ceph node-exporter.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:35:44.070 INFO:journalctl@ceph.node-exporter.a.smithi175.stdout:May 25 06:35:43 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.a.service: Consumed 3.096s CPU time. 2024-05-25T06:35:44.362 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:44 smithi175 systemd[1]: Stopping Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:35:44.362 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:44 smithi175 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a[78201]: ts=2024-05-25T06:35:44.322Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-05-25T06:35:44.639 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:44 smithi175 podman[137458]: 2024-05-25 06:35:44.363340788 +0000 UTC m=+0.132110571 container died 12c8cd2eac3288ddd0c1080a93cb114c5d5daea9b2a7a569000caa88f52628d8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:44.989 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:44 smithi175 podman[137458]: 2024-05-25 06:35:44.739108893 +0000 UTC m=+0.507878654 container cleanup 12c8cd2eac3288ddd0c1080a93cb114c5d5daea9b2a7a569000caa88f52628d8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:44.989 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:44 smithi175 bash[137458]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a 2024-05-25T06:35:45.319 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:44 smithi175 podman[137472]: 2024-05-25 06:35:44.990046938 +0000 UTC m=+0.621079528 container remove 12c8cd2eac3288ddd0c1080a93cb114c5d5daea9b2a7a569000caa88f52628d8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-25T06:35:45.319 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:45 smithi175 podman[137472]: 2024-05-25 06:35:45.065698733 +0000 UTC m=+0.696731301 volume remove 1f9a54bdf8ee3ec615985e99315ccd1985c777a526df21de5d5afdee1e2b06c5 2024-05-25T06:35:45.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:45 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-25T06:35:45.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:45 smithi175 systemd[1]: Stopped Ceph alertmanager.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:35:45.320 INFO:journalctl@ceph.alertmanager.a.smithi175.stdout:May 25 06:35:45 smithi175 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@alertmanager.a.service: Consumed 1.660s CPU time. 2024-05-25T06:36:08.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 --force --keep-logs 2024-05-25T06:36:09.785 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:09 smithi179 systemd[1]: Stopping Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:36:10.102 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:09 smithi179 podman[102005]: 2024-05-25 06:36:09.830191347 +0000 UTC m=+0.126961313 container died a3e7fafe20dff24a37ef15276ba93d6b82e08a7418670db640ffb46968833d3e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:36:10.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:10 smithi179 podman[102005]: 2024-05-25 06:36:10.277814233 +0000 UTC m=+0.574584206 container cleanup a3e7fafe20dff24a37ef15276ba93d6b82e08a7418670db640ffb46968833d3e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:36:10.535 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:10 smithi179 bash[102005]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b 2024-05-25T06:36:10.874 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:10 smithi179 podman[102018]: 2024-05-25 06:36:10.553983161 +0000 UTC m=+0.728083318 container remove a3e7fafe20dff24a37ef15276ba93d6b82e08a7418670db640ffb46968833d3e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-25T06:36:10.874 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:10 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-25T06:36:10.875 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:10 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-25T06:36:10.875 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:10 smithi179 systemd[1]: Stopped Ceph node-exporter.b for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:36:10.875 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:May 25 06:36:10 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@node-exporter.b.service: Consumed 3.092s CPU time. 2024-05-25T06:36:11.134 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:10 smithi179 systemd[1]: Stopping Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968... 2024-05-25T06:36:11.134 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=server t=2024-05-25T06:36:11.12431423Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-05-25T06:36:11.134 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=ticker t=2024-05-25T06:36:11.124678843Z level=info msg=stopped last_tick=2024-05-25T06:36:10Z 2024-05-25T06:36:11.134 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a[65985]: logger=tracing t=2024-05-25T06:36:11.124718091Z level=info msg="Closing tracing" 2024-05-25T06:36:11.447 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 podman[102128]: 2024-05-25 06:36:11.134831296 +0000 UTC m=+0.118292840 container died fb91afb62378e4509d748d48238025bcf84de4ef3961ae5d642952906365574f (image=quay.io/ceph/grafana:9.4.12, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-25T06:36:11.447 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 podman[102128]: 2024-05-25 06:36:11.4486045 +0000 UTC m=+0.432066043 container cleanup fb91afb62378e4509d748d48238025bcf84de4ef3961ae5d642952906365574f (image=quay.io/ceph/grafana:9.4.12, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-25T06:36:11.785 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 bash[102128]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a 2024-05-25T06:36:11.785 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 podman[102140]: 2024-05-25 06:36:11.658273867 +0000 UTC m=+0.521569994 container remove fb91afb62378e4509d748d48238025bcf84de4ef3961ae5d642952906365574f (image=quay.io/ceph/grafana:9.4.12, name=ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-25T06:36:12.285 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-25T06:36:12.285 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 systemd[1]: Stopped Ceph grafana.a for 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968. 2024-05-25T06:36:12.285 INFO:journalctl@ceph.grafana.a.smithi179.stdout:May 25 06:36:11 smithi179 systemd[1]: ceph-0bfa7ec4-1a5b-11ef-bc9b-c7b262605968@grafana.a.service: Consumed 11.085s CPU time. 2024-05-25T06:36:23.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-25T06:36:23.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-25T06:36:23.580 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-25T06:36:23.582 DEBUG:teuthology.misc:Transferring archived files from smithi175:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343/remote/smithi175/crash 2024-05-25T06:36:23.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/crash -- . 2024-05-25T06:36:23.633 INFO:teuthology.orchestra.run.smithi175.stderr:tar: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/crash: Cannot open: No such file or directory 2024-05-25T06:36:23.634 INFO:teuthology.orchestra.run.smithi175.stderr:tar: Error is not recoverable: exiting now 2024-05-25T06:36:23.635 DEBUG:teuthology.misc:Transferring archived files from smithi179:/var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343/remote/smithi179/crash 2024-05-25T06:36:23.636 DEBUG:teuthology.orchestra.run.smithi179:> sudo tar c -f - -C /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/crash -- . 2024-05-25T06:36:23.669 INFO:teuthology.orchestra.run.smithi179.stderr:tar: /var/lib/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/crash: Cannot open: No such file or directory 2024-05-25T06:36:23.670 INFO:teuthology.orchestra.run.smithi179.stderr:tar: Error is not recoverable: exiting now 2024-05-25T06:36:23.671 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-25T06:36:23.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-25T06:36:23.704 INFO:teuthology.orchestra.run.smithi175.stdout:2024-05-25T06:17:13.243509+0000 mon.a (mon.0) 1057 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:36:23.705 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-05-25T06:36:23.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo egrep '\[SEC\]' /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-25T06:36:23.773 DEBUG:teuthology.orchestra.run.smithi175:> sudo egrep '\[ERR\]' /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-25T06:36:23.840 DEBUG:teuthology.orchestra.run.smithi175:> sudo egrep '\[WRN\]' /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-25T06:36:23.908 INFO:teuthology.orchestra.run.smithi175.stdout:2024-05-25T06:17:13.243509+0000 mon.a (mon.0) 1057 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-25T06:36:23.908 INFO:tasks.cephadm:Compressing logs... 2024-05-25T06:36:23.908 DEBUG:teuthology.orchestra.run.smithi175:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-25T06:36:23.950 DEBUG:teuthology.orchestra.run.smithi179:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-25T06:36:23.972 INFO:teuthology.orchestra.run.smithi175.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-25T06:36:23.972 INFO:teuthology.orchestra.run.smithi175.stderr:: No such file or directory 2024-05-25T06:36:23.972 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log 2024-05-25T06:36:23.973 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.2.log 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.audit.log 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-client.rgw.foo.smithi175.iwhmky.log 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.3.log 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.audit.log: /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-client.rgw.foo.smithi175.iwhmky.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.1.log 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr: 75.9% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-client.rgw.foo.smithi175.iwhmky.log.gz 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mgr.y.log 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.0.log 2024-05-25T06:36:23.975 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.1.log: /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.c.log 2024-05-25T06:36:23.976 INFO:teuthology.orchestra.run.smithi179.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-25T06:36:23.976 INFO:teuthology.orchestra.run.smithi179.stderr:: No such file or directory 2024-05-25T06:36:23.977 INFO:teuthology.orchestra.run.smithi179.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log 2024-05-25T06:36:23.977 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.audit.log 2024-05-25T06:36:23.977 INFO:teuthology.orchestra.run.smithi179.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-client.rgw.foo.smithi179.hopgfy.log 2024-05-25T06:36:23.977 INFO:teuthology.orchestra.run.smithi179.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.7.log 2024-05-25T06:36:23.977 INFO:teuthology.orchestra.run.smithi179.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.5.log 2024-05-25T06:36:23.979 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log: /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.audit.log: /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-client.rgw.foo.smithi179.hopgfy.log: /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.4.log 2024-05-25T06:36:23.979 INFO:teuthology.orchestra.run.smithi179.stderr: 76.0% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-client.rgw.foo.smithi179.hopgfy.log.gz 2024-05-25T06:36:23.979 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mgr.x.log 2024-05-25T06:36:23.979 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.6.log 2024-05-25T06:36:23.979 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.cephadm.log 2024-05-25T06:36:23.981 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.6.log: 89.2% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log.gz 2024-05-25T06:36:23.981 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.0.log: 93.9% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.log.gz 2024-05-25T06:36:23.981 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.a.log 2024-05-25T06:36:23.982 INFO:teuthology.orchestra.run.smithi179.stderr: 91.4% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.audit.log.gz 2024-05-25T06:36:23.984 INFO:teuthology.orchestra.run.smithi179.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-volume.log 2024-05-25T06:36:23.984 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.b.log 2024-05-25T06:36:23.986 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.c.log: 94.6% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.audit.log.gz 2024-05-25T06:36:23.986 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.cephadm.log 2024-05-25T06:36:23.988 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-volume.log: 96.0% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.cephadm.log.gz 2024-05-25T06:36:23.990 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.a.log: 92.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-25T06:36:23.999 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/tcmu-runner.log 2024-05-25T06:36:24.000 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.b.log: 91.0% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-volume.log.gz 2024-05-25T06:36:24.007 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.cephadm.log: 97.1% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph.cephadm.log.gz 2024-05-25T06:36:24.008 INFO:teuthology.orchestra.run.smithi179.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-25T06:36:24.013 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-volume.log 2024-05-25T06:36:24.014 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/tcmu-runner.log: 88.9% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/tcmu-runner.log.gz 2024-05-25T06:36:24.031 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-volume.log: 91.0% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-volume.log.gz 2024-05-25T06:36:24.127 INFO:teuthology.orchestra.run.smithi179.stderr: 89.8% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mgr.x.log.gz 2024-05-25T06:36:24.505 INFO:teuthology.orchestra.run.smithi175.stderr: 89.0% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mgr.y.log.gz 2024-05-25T06:36:24.550 INFO:teuthology.orchestra.run.smithi179.stderr: 92.5% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.b.log.gz 2024-05-25T06:36:24.807 INFO:teuthology.orchestra.run.smithi175.stderr: 92.4% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.c.log.gz 2024-05-25T06:36:25.449 INFO:teuthology.orchestra.run.smithi179.stderr: 93.2% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.6.log.gz 2024-05-25T06:36:25.682 INFO:teuthology.orchestra.run.smithi175.stderr: 91.1% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-mon.a.log.gz 2024-05-25T06:36:25.738 INFO:teuthology.orchestra.run.smithi175.stderr: 93.3% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.2.log.gz 2024-05-25T06:36:25.832 INFO:teuthology.orchestra.run.smithi175.stderr: 93.5% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.0.log.gz 2024-05-25T06:36:25.885 INFO:teuthology.orchestra.run.smithi175.stderr: 93.5% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.1.log.gz 2024-05-25T06:36:25.909 INFO:teuthology.orchestra.run.smithi179.stderr: 93.4% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.5.log.gz 2024-05-25T06:36:26.007 INFO:teuthology.orchestra.run.smithi179.stderr: 93.8% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.7.log.gz 2024-05-25T06:36:26.245 INFO:teuthology.orchestra.run.smithi179.stderr: 93.6% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.4.log.gz 2024-05-25T06:36:26.247 INFO:teuthology.orchestra.run.smithi179.stderr: 2024-05-25T06:36:26.248 INFO:teuthology.orchestra.run.smithi179.stderr:real 0m2.283s 2024-05-25T06:36:26.248 INFO:teuthology.orchestra.run.smithi179.stderr:user 0m8.197s 2024-05-25T06:36:26.248 INFO:teuthology.orchestra.run.smithi179.stderr:sys 0m0.290s 2024-05-25T06:36:26.720 INFO:teuthology.orchestra.run.smithi175.stderr: 93.4% -- replaced with /var/log/ceph/0bfa7ec4-1a5b-11ef-bc9b-c7b262605968/ceph-osd.3.log.gz 2024-05-25T06:36:26.723 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-05-25T06:36:26.723 INFO:teuthology.orchestra.run.smithi175.stderr:real 0m2.760s 2024-05-25T06:36:26.723 INFO:teuthology.orchestra.run.smithi175.stderr:user 0m10.992s 2024-05-25T06:36:26.723 INFO:teuthology.orchestra.run.smithi175.stderr:sys 0m0.407s 2024-05-25T06:36:26.723 INFO:tasks.cephadm:Archiving logs... 2024-05-25T06:36:26.723 DEBUG:teuthology.misc:Transferring archived files from smithi175:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343/remote/smithi175/log 2024-05-25T06:36:26.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-25T06:36:28.202 DEBUG:teuthology.misc:Transferring archived files from smithi179:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343/remote/smithi179/log 2024-05-25T06:36:28.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-25T06:36:29.409 INFO:tasks.cephadm:Removing cluster... 2024-05-25T06:36:29.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 --force 2024-05-25T06:36:29.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0bfa7ec4-1a5b-11ef-bc9b-c7b262605968 --force 2024-05-25T06:36:30.000 INFO:tasks.cephadm:Removing cephadm ... 2024-05-25T06:36:30.001 DEBUG:teuthology.orchestra.run.smithi175:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-25T06:36:30.017 DEBUG:teuthology.orchestra.run.smithi179:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-25T06:36:30.032 INFO:tasks.cephadm:Teardown complete 2024-05-25T06:36:30.033 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-25T06:36:30.042 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-25T06:36:30.043 DEBUG:teuthology.orchestra.run.smithi175:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-25T06:36:30.059 DEBUG:teuthology.orchestra.run.smithi179:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-25T06:36:30.072 INFO:teuthology.orchestra.run.smithi175.stderr:bash: line 1: ntpq: command not found 2024-05-25T06:36:30.076 INFO:teuthology.orchestra.run.smithi175.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-25T06:36:30.076 INFO:teuthology.orchestra.run.smithi175.stdout:=============================================================================== 2024-05-25T06:36:30.076 INFO:teuthology.orchestra.run.smithi175.stdout:^* hv01.front.sepia.ceph.com 3 6 377 11 +1102us[+1105us] +/- 66ms 2024-05-25T06:36:30.077 INFO:teuthology.orchestra.run.smithi175.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 15 -528us[ -526us] +/- 63ms 2024-05-25T06:36:30.077 INFO:teuthology.orchestra.run.smithi175.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 16 -637us[ -635us] +/- 76ms 2024-05-25T06:36:30.077 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T06:36:30.087 INFO:teuthology.orchestra.run.smithi179.stderr:bash: line 1: ntpq: command not found 2024-05-25T06:36:30.091 INFO:teuthology.orchestra.run.smithi179.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-25T06:36:30.091 INFO:teuthology.orchestra.run.smithi179.stdout:=============================================================================== 2024-05-25T06:36:30.091 INFO:teuthology.orchestra.run.smithi179.stdout:^* hv01.front.sepia.ceph.com 3 7 377 16 +1040us[+1041us] +/- 66ms 2024-05-25T06:36:30.091 INFO:teuthology.orchestra.run.smithi179.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 79 -574us[ -573us] +/- 63ms 2024-05-25T06:36:30.091 INFO:teuthology.orchestra.run.smithi179.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 19 -707us[ -706us] +/- 76ms 2024-05-25T06:36:30.091 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-25T06:36:30.092 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-25T06:36:30.100 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-25T06:36:30.101 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-25T06:36:30.110 DEBUG:teuthology.orchestra.run.smithi175:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-25T06:36:30.209 DEBUG:teuthology.orchestra.run.smithi179:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-25T06:36:30.285 DEBUG:teuthology.orchestra.run.smithi175:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-25T06:36:30.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T06:36:30.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-25T06:36:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T06:36:30.343 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-25T06:36:30.355 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-25T06:36:30.364 INFO:teuthology.task.internal:Duration was 2874.736488 seconds 2024-05-25T06:36:30.364 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-25T06:36:30.372 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-25T06:36:30.373 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-25T06:36:30.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-25T06:36:30.410 INFO:teuthology.orchestra.run.smithi175.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-25T06:36:30.421 INFO:teuthology.orchestra.run.smithi179.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-25T06:36:30.859 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-25T06:36:30.859 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi175.front.sepia.ceph.com 2024-05-25T06:36:30.859 DEBUG:teuthology.orchestra.run.smithi175:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-25T06:36:30.919 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi179.front.sepia.ceph.com 2024-05-25T06:36:30.919 DEBUG:teuthology.orchestra.run.smithi179:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-25T06:36:30.944 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-25T06:36:30.944 DEBUG:teuthology.orchestra.run.smithi175:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-25T06:36:30.961 DEBUG:teuthology.orchestra.run.smithi179:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-25T06:36:31.009 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-25T06:36:31.010 DEBUG:teuthology.orchestra.run.smithi175:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-25T06:36:31.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-25T06:36:31.679 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-25T06:36:31.688 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-25T06:36:31.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-25T06:36:31.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-25T06:36:31.738 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-25T06:36:31.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-25T06:36:31.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-25T06:36:31.777 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern = core 2024-05-25T06:36:31.801 INFO:teuthology.orchestra.run.smithi179.stdout:kernel.core_pattern = core 2024-05-25T06:36:31.839 DEBUG:teuthology.orchestra.run.smithi175:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-25T06:36:31.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T06:36:31.858 DEBUG:teuthology.orchestra.run.smithi179:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-25T06:36:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-25T06:36:31.894 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-25T06:36:31.903 INFO:teuthology.task.internal:Transferring archived files... 2024-05-25T06:36:31.904 DEBUG:teuthology.misc:Transferring archived files from smithi175:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343/remote/smithi175 2024-05-25T06:36:31.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-25T06:36:31.956 DEBUG:teuthology.misc:Transferring archived files from smithi179:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-05-24_21:08:02-orch-squid-distro-default-smithi/7725343/remote/smithi179 2024-05-25T06:36:31.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-25T06:36:31.998 INFO:teuthology.task.internal:Removing archive directory... 2024-05-25T06:36:31.998 DEBUG:teuthology.orchestra.run.smithi175:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-25T06:36:32.001 DEBUG:teuthology.orchestra.run.smithi179:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-25T06:36:32.046 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-25T06:36:32.059 INFO:teuthology.task.internal:Not uploading archives. 2024-05-25T06:36:32.059 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-25T06:36:32.068 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-25T06:36:32.069 DEBUG:teuthology.orchestra.run.smithi175:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-25T06:36:32.071 DEBUG:teuthology.orchestra.run.smithi179:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-25T06:36:32.083 INFO:teuthology.orchestra.run.smithi175.stdout: 265517 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 25 06:36 /home/ubuntu/cephtest 2024-05-25T06:36:32.101 INFO:teuthology.orchestra.run.smithi179.stdout: 265953 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 25 06:36 /home/ubuntu/cephtest 2024-05-25T06:36:32.102 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-25T06:36:32.112 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-25T06:36:32.134 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} duration: 2874.736488342285 failure_reason: '"2024-05-25T06:17:13.243509+0000 mon.a (mon.0) 1057 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-05-25T06:36:32.134 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-25T06:36:32.218 INFO:teuthology.run:FAIL